python如何去除異常值和缺失值的插值
1.使用箱型法去除異常值:
import numpy as np import pandas as pd import matplotlib as plt import os data = pd.read_excel('try.xls', header=0) # print(data.shape) # print(data.head(10)) # print(data.describe()) neg_list = ['位移'] print("(1)數據的行數為:") R = data.shape[0] print(R) print("(2)小於或大於閾值的數據提取:") for item in neg_list: neg_item = data[item]<2000 print(item + '小於2000的有' + str(neg_item.sum()) + '個') print("(3)異常值的個數:") for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) q_abnormal_L = data[item] < data[item].quantile(0.25) - 1.5 * iqr q_abnormal_U = data[item] > data[item].quantile(0.75) + 1.5 * iqr print(item + '中有' + str(q_abnormal_L.sum() + q_abnormal_U.sum()) + '個異常值') print("(4)箱型圖確定上下限:") for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) Too_small = data[item].quantile(0.25) - 1.5 * iqr Too_big = data[item].quantile(0.25) + 1.5 * iqr print("下限是", Too_small) print("上限是", Too_big ) print("(5)所有數據為:") a = [] for i in neg_list: a.append(data[i]) print(a) print("(6)所有正常數據:") b = [] j = 0 while j < R: if (a[0][j] > Too_small): if (a[0][j] < Too_big): b.append(a[0][j]) j += 1 print(b) print("(7)所有異常數據:") c = [] i = 0 while i < R: if (a[0][i] < Too_small or a[0][i] > Too_big): c.append(a[0][i]) a[0][i] = None i +=1 print(c) print("(8)把所有異常數據刪除後:") print(a) print("(9)所有數據處理後輸出:") d = [] k = 0 while k < R: d.append(a[0][k]) k +=1 print(d) df = pd.DataFrame(d,columns= ['位移']) df.to_excel("try_result.xls")
2.拉格朗日插值:
import os import pandas as pd import numpy as np from scipy.interpolate import lagrange import matplotlib.pyplot as plt plt.rcParams['font.sans-serif']=['SimHei'] #用來正常顯示中文標簽 plt.rcParams['axes.unicode_minus']=False #用來正常顯示負號 # 數據的讀取 data = pd.read_excel('try.xls', header=0) neg_list = ['位移'] # 數據的行數 R = data.shape[0] # 異常數據的個數 for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) q_abnormal_L = data[item] < data[item].quantile(0.25) - 1.5 * iqr q_abnormal_U = data[item] > data[item].quantile(0.75) + 1.5 * iqr # print(item + '中有' + str(q_abnormal_L.sum() + q_abnormal_U.sum()) + '個異常值') # 確定數據上限和下限 for item in neg_list: iqr = data[item].quantile(0.75) - data[item].quantile(0.25) Too_small = data[item].quantile(0.25) - 1.5 * iqr Too_big = data[item].quantile(0.25) + 1.5 * iqr data[u'位移'][(data[u'位移']<Too_small) | (data[u'位移']>Too_big)] = None #過濾異常值,將其變為空值 #s為列向量,n為被插值位置,k為取前後的數據個數 def ployinter(s,n,k=5): y = s[list(range(n-k,n)) + list(range(n+1,n+1+k))] y = y[y.notnull()] #剔除空值 return lagrange(y.index,list(y))(n) #逐個元素判斷是否需要插值 for i in data.columns: for j in range(len(data)): if(data[i].isnull())[j]: data[i][j] = ployinter(data[i],j) # print(data[u'位移']) # 輸出拉格朗日插值後的數據 data.to_excel("try_result.xls") # 把表格列數據調整為arr,arr為修改後的數據 print("拉格朗日插值後的數據:") d = [] k = 0 while k < R: d.append(data[u'位移'][k]) k +=1 # print(d) arr = np.array(d) print(arr) # 輸出圖像 x = np.arange(len(d)) plt.plot(x,d,'b-',label="one", marker='*',markersize=4,linewidth=1) # b代表blue顏色 -代表直線 plt.title('位移曲線') plt.legend(loc='upper left',bbox_to_anchor=(1.0,1.0)) # 直接更改X軸坐標數 # plt.xticks((0,1,2,3,4,5,6,7,8),('0', '1', '2', '3', '4', '5', '6', '7', '8')) plt.xlabel('時間/h') plt.ylabel('位移/mm') #plt.grid(x1) plt.show()
3.數據擬合:
import numpy as np import pandas as pd import matplotlib.pyplot as plt from scipy.optimize import leastsq def Fun(p, x): # 定義擬合函數形式 a1, a2, a3 , a4 = p return a1 * x ** 3 + a2 * x ** 2 + a3 * x + a4 def error(p, x, y): # 擬合殘差 return Fun(p, x) - y def main(): x = np.linspace(1, 31, 31) # 創建時間序列 data = pd.read_excel('try.xls', header=0) y = data[u'位移'] p0 = [0.1, -0.01, 100, 1000] # 擬合的初始參數設置 para = leastsq(error, p0, args=(x, y)) # 進行擬合 y_fitted = Fun(para[0], x) # 畫出擬合後的曲線 plt.figure plt.plot(x, y, 'r', label='Original curve') plt.plot(x, y_fitted, '-b', label='Fitted curve') plt.legend() plt.show() print(para[0]) if __name__ == '__main__': main()
4.輸出圖像:
import pandas as pd import numpy as np import matplotlib.pyplot as plt plt.rcParams['font.sans-serif']=['SimHei'] #用來正常顯示中文標簽 plt.rcParams['axes.unicode_minus']=False #用來正常顯示負號 jiaodu = ['0', '15', '30', '15', '60', '75', '90', '105', '120'] x = range(len(jiaodu)) y = [85.6801, 7.64586, 86.0956, 159.229, 179.534, 163.238, 96.4436, 10.1619, 90.9262,] #plt.figure(figsize=(10, 6)) plt.plot(x,y,'b-',label="1", marker='*',markersize=7,linewidth=3) # b代表blue顏色 -代表直線 plt.title('各個區域亮度變化') plt.legend(loc='upper left',bbox_to_anchor=(1.0,1.0)) plt.xticks((0,1,2,3,4,5,6,7,8),('0', '15', '30', '15', '60', '75', '90', '105', '120')) plt.xlabel('角度') plt.ylabel('亮度') #plt.grid(x1) plt.show()
到此這篇關於python如何去除異常值和缺失值的插值的文章就介紹到這瞭,更多相關python去除異常值和缺失值內容請搜索WalkonNet以前的文章或繼續瀏覽下面的相關文章希望大傢以後多多支持WalkonNet!
推薦閱讀:
- python數據可視化之matplotlib.pyplot基礎以及折線圖
- Python畫圖常用命令大全(詳解)
- Python數據分析應用之Matplotlib數據可視化詳情
- python數學建模之Matplotlib 實現圖片繪制
- 利用python繪制線型圖