基于PySpark的网络服务异常检测系统 阶段总结(三)

本篇博文主要介绍如何使用SVDD算法和Isolation Forest算法来进行异常检测

首先是SVDD算法，主要是用sklearn里面的svm.OneClassSVM()来做单分类的异常检测，用cross_validation作为交叉验证调参

import numpy as np
from sklearn import svm
from sklearn.cross_validation import train_test_split

class Calculator_timeout(object):
    def __init__(self,Appid):
        self.Appid = Appid
        #这里的self.respondse_list和self.request_data_list分别是待训练的数据集列表  我这里是从MySQL数据库中通过SparkSQL取的，后面会详细介绍  
        self.X1 = np.array(self.respondse_list)
        self.y1 = np.array(self.request_data_list)
        self.y = range(0, len(self.X1))
        self.X = np.c_[self.X1, self.y1]
        self.X_testt = self.X[len(self.X)-360:]

def get_best_parameters(self):
        X_train, X_test, y_train, y_test = train_test_split(
            self.X, self.y, test_size=0.5, random_state=42)
        parameters = {"nu": [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
                      "gamma": [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
                      "cache_size": [200],
                      "kernel": ['rbf', 'linear'],
                      "max_iter": [-1],
                      "shrinking": [True, False],
                      "tol": [0.001],
                      "verbose": [False]
                      }

        svr = svm.OneClassSVM()
        clf = GridSearchCV(sc, svr, parameters, scoring='%s_weighted' % 'precision')
        clf.fit(X_train, y_train)
        print("Best parameters set found on development set:")
        print(clf.best_params_)
        return clf.best_params_

    def get_pic(self):
        aaa = self.get_best_parameters()
        clf = svm.OneClassSVM(kernel = aaa['kernel'],verbose = aaa['verbose'],
                              max_iter = aaa['max_iter'], tol = aaa['tol'],
                              cache_size = aaa['cache_size'], shrinking = aaa['shrinking'],
                              nu = aaa['nu'], gamma = aaa['gamma'])

        clf.fit(self.X)
        # pred_train = clf.predict(self.X,self.y)
        pred_test = clf.predict(self.X_testt)
        print self.X1
        print len(self.respondse_list)
        print ("novelty detection result:",pred_test)
        return pred_test,self.X_testt

 下一篇将介绍使用SparkSQL和mllib实现kmeans算法预测