Data with Python - Examples

Transcription

1 Data with Python - Examples May 5, 2018 In [57]: #ipython In [58]: import pandas as pd 1 pandas: load data In [59]: DATA_PATH = '/usr/lib/python3/dist-packages/pandas/tests/data/tips.csv' In [60]: #!cat /usr/lib/python3/dist-packages/pandas/tests/data/tips.csv In [61]: data = pd.read_csv(data_path) In [62]: data Out[62]: total_bill tip sex smoker day time size Female No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner Male No Sun Dinner Female No Sun Dinner Male No Sun Dinner Female No Sun Dinner Male No Sat Dinner Male No Sat Dinner Female No Sat Dinner 2 1

2 Female No Sat Dinner Male No Sat Dinner Male No Sat Dinner Male No Sat Dinner Male No Sat Dinner Male No Sat Dinner Male No Sat Dinner Female No Sat Dinner Female Yes Sat Dinner Female Yes Sat Dinner Male Yes Sat Dinner Male Yes Sat Dinner Male Yes Sat Dinner Female Yes Sat Dinner Male Yes Fri Lunch Female Yes Fri Lunch Male Yes Fri Lunch Female No Fri Lunch Male Yes Fri Lunch Female Yes Fri Lunch Female Yes Fri Lunch Male No Sat Dinner Male No Sat Dinner Female Yes Sat Dinner Male Yes Sat Dinner Male Yes Sat Dinner Male No Sat Dinner Male No Sat Dinner Male Yes Sat Dinner Male No Sat Dinner Male Yes Sat Dinner Male Yes Sat Dinner Female No Sat Dinner Male No Sat Dinner Female Yes Sat Dinner Male Yes Sat Dinner Male No Sat Dinner Female No Thur Dinner 2 [244 rows x 7 columns] In [63]: data.sort_values('tip') Out[63]: total_bill tip sex smoker day time size Female Yes Sat Dinner Male Yes Sat Dinner Female Yes Fri Dinner 2 2

3 Female No Sat Dinner Female No Sun Dinner Female Yes Sat Dinner Male Yes Sat Dinner Male No Sat Dinner Male No Sat Dinner Female No Thur Lunch Male No Sun Dinner Female No Thur Lunch Male Yes Sat Dinner Male No Thur Lunch Male No Sat Dinner Male No Sat Dinner Male No Thur Lunch Male Yes Sun Dinner Female No Sat Dinner Male Yes Fri Dinner Male No Fri Dinner Female No Thur Lunch Female No Thur Lunch Male Yes Sat Dinner Female No Thur Lunch Male No Thur Lunch Male No Sun Dinner Male No Sun Dinner Male Yes Fri Lunch Female Yes Sat Dinner Male No Sun Dinner Male Yes Fri Dinner Male No Sun Dinner Male No Sat Dinner Male No Thur Lunch Female No Thur Lunch Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner Male Yes Thur Lunch Female Yes Thur Lunch Female Yes Sat Dinner Male No Sun Dinner Female No Sun Dinner Male Yes Sun Dinner Male Yes Sat Dinner Female No Thur Lunch Female No Sun Dinner Male No Sun Dinner Male Yes Sun Dinner 2 3

4 Male No Thur Lunch Male No Sat Dinner Male No Sun Dinner Male Yes Sun Dinner Female Yes Sat Dinner Male No Thur Lunch Male No Sat Dinner Male No Sat Dinner Male No Sat Dinner Male Yes Sat Dinner 3 [244 rows x 7 columns] In [64]: data.head() Out[64]: total_bill tip sex smoker day time size Female No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner 4 In [65]: data['tip'] Out[65]:

5 Name: tip, Length: 244, dtype: float64 In [66]: data['tip'] / data['total_bill'] Out[66]:

6

7 Length: 244, dtype: float64 In [67]: data['perc_tip'] = data['tip'] / data['total_bill'] In [68]: data.head() Out[68]: total_bill tip sex smoker day time size perc_tip Female No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner In [69]: data[data.sex == 'Female'] Out[69]: total_bill tip sex smoker day time size perc_tip Female No Sun Dinner Female No Sun Dinner Female No Sun Dinner Female No Sun Dinner Female No Sun Dinner Female No Sun Dinner Female No Sat Dinner Female No Sat Dinner Female No Sat Dinner Female No Sat Dinner Female No Sat Dinner Female No Sat Dinner Female No Sun Dinner Female No Sun Dinner Female No Sat Dinner Female No Sat Dinner Female Yes Sat Dinner Female No Sat Dinner Female Yes Sat Dinner Female Yes Sat Dinner Female No Sat Dinner Female No Thur Lunch Female No Thur Lunch Female Yes Fri Dinner Female Yes Fri Dinner Female No Fri Dinner Female Yes Fri Dinner Female Yes Fri Dinner

8 Female Yes Sat Dinner Female Yes Sat Dinner Female No Sun Dinner Female No Sun Dinner Female No Sun Dinner Female No Sun Dinner Female Yes Sun Dinner Female Yes Sat Dinner Female Yes Sat Dinner Female Yes Sun Dinner Female Yes Sun Dinner Female Yes Sun Dinner Female Yes Thur Lunch Female Yes Thur Lunch Female Yes Thur Lunch Female Yes Thur Lunch Female Yes Thur Lunch Female Yes Thur Lunch Female Yes Thur Lunch Female Yes Sat Dinner Female Yes Sat Dinner Female Yes Sat Dinner Female Yes Sat Dinner Female Yes Sat Dinner Female Yes Fri Lunch Female No Fri Lunch Female Yes Fri Lunch Female Yes Fri Lunch Female Yes Sat Dinner Female No Sat Dinner Female Yes Sat Dinner Female No Thur Dinner [87 rows x 8 columns] In [70]: data[data.sex == 'Female'].to_csv('waitresses.csv') In [71]:!cat waitresses.csv,total_bill,tip,sex,smoker,day,time,size,perc_tip 0,16.99,1.01,Female,No,Sun,Dinner,2, ,24.59,3.61,Female,No,Sun,Dinner,4, ,35.26,5.0,Female,No,Sun,Dinner,4, ,14.83,3.02,Female,No,Sun,Dinner,2, ,10.33,1.67,Female,No,Sun,Dinner,3, ,16.97,3.5,Female,No,Sun,Dinner,3, ,20.29,2.75,Female,No,Sat,Dinner,2,

9 22,15.77,2.23,Female,No,Sat,Dinner,2, ,19.65,3.0,Female,No,Sat,Dinner,2, ,15.06,3.0,Female,No,Sat,Dinner,2, ,20.69,2.45,Female,No,Sat,Dinner,4, ,16.93,3.07,Female,No,Sat,Dinner,3, ,10.29,2.6,Female,No,Sun,Dinner,2, ,34.81,5.2,Female,No,Sun,Dinner,4, ,26.41,1.5,Female,No,Sat,Dinner,2, ,16.45,2.47,Female,No,Sat,Dinner,2, ,3.07,1.0,Female,Yes,Sat,Dinner,1, ,17.07,3.0,Female,No,Sat,Dinner,3, ,26.86,3.14,Female,Yes,Sat,Dinner,2, ,25.28,5.0,Female,Yes,Sat,Dinner,2, ,14.73,2.2,Female,No,Sat,Dinner,2, ,10.07,1.83,Female,No,Thur,Lunch,1, ,34.83,5.17,Female,No,Thur,Lunch,4, ,5.75,1.0,Female,Yes,Fri,Dinner,2, ,16.32,4.3,Female,Yes,Fri,Dinner,2, ,22.75,3.25,Female,No,Fri,Dinner,2, ,11.35,2.5,Female,Yes,Fri,Dinner,2, ,15.38,3.0,Female,Yes,Fri,Dinner,2, ,44.3,2.5,Female,Yes,Sat,Dinner,3, ,22.42,3.48,Female,Yes,Sat,Dinner,2, ,20.92,4.08,Female,No,Sat,Dinner,2, ,14.31,4.0,Female,Yes,Sat,Dinner,2, ,7.25,1.0,Female,No,Sat,Dinner,1, ,25.71,4.0,Female,No,Sun,Dinner,3, ,17.31,3.5,Female,No,Sun,Dinner,2, ,10.65,1.5,Female,No,Thur,Lunch,2, ,12.43,1.8,Female,No,Thur,Lunch,2, ,24.08,2.92,Female,No,Thur,Lunch,4, ,13.42,1.68,Female,No,Thur,Lunch,2, ,12.48,2.52,Female,No,Thur,Lunch,2, ,29.8,4.2,Female,No,Thur,Lunch,6, ,14.52,2.0,Female,No,Thur,Lunch,2, ,11.38,2.0,Female,No,Thur,Lunch,2, ,20.27,2.83,Female,No,Thur,Lunch,2, ,11.17,1.5,Female,No,Thur,Lunch,2, ,12.26,2.0,Female,No,Thur,Lunch,2, ,18.26,3.25,Female,No,Thur,Lunch,2, ,8.51,1.25,Female,No,Thur,Lunch,2, ,10.33,2.0,Female,No,Thur,Lunch,2, ,14.15,2.0,Female,No,Thur,Lunch,2, ,13.16,2.75,Female,No,Thur,Lunch,2, ,17.47,3.5,Female,No,Thur,Lunch,2, ,27.05,5.0,Female,No,Thur,Lunch,6, ,16.43,2.3,Female,No,Thur,Lunch,2, ,8.35,1.5,Female,No,Thur,Lunch,2,

10 146,18.64,1.36,Female,No,Thur,Lunch,3, ,11.87,1.63,Female,No,Thur,Lunch,2, ,29.85,5.14,Female,No,Sun,Dinner,5, ,25.0,3.75,Female,No,Sun,Dinner,4, ,13.39,2.61,Female,No,Sun,Dinner,2, ,16.21,2.0,Female,No,Sun,Dinner,3, ,17.51,3.0,Female,Yes,Sun,Dinner,2, ,10.59,1.61,Female,Yes,Sat,Dinner,2, ,10.63,2.0,Female,Yes,Sat,Dinner,2, ,9.6,4.0,Female,Yes,Sun,Dinner,2, ,20.9,3.5,Female,Yes,Sun,Dinner,3, ,18.15,3.5,Female,Yes,Sun,Dinner,3, ,19.81,4.19,Female,Yes,Thur,Lunch,2, ,43.11,5.0,Female,Yes,Thur,Lunch,4, ,13.0,2.0,Female,Yes,Thur,Lunch,2, ,12.74,2.01,Female,Yes,Thur,Lunch,2, ,13.0,2.0,Female,Yes,Thur,Lunch,2, ,16.4,2.5,Female,Yes,Thur,Lunch,2, ,16.47,3.23,Female,Yes,Thur,Lunch,3, ,12.76,2.23,Female,Yes,Sat,Dinner,2, ,13.27,2.5,Female,Yes,Sat,Dinner,2, ,28.17,6.5,Female,Yes,Sat,Dinner,3, ,12.9,1.1,Female,Yes,Sat,Dinner,2, ,30.14,3.09,Female,Yes,Sat,Dinner,4, ,13.42,3.48,Female,Yes,Fri,Lunch,2, ,15.98,3.0,Female,No,Fri,Lunch,3, ,16.27,2.5,Female,Yes,Fri,Lunch,2, ,10.09,2.0,Female,Yes,Fri,Lunch,2, ,22.12,2.88,Female,Yes,Sat,Dinner,2, ,35.83,4.67,Female,No,Sat,Dinner,3, ,27.18,2.0,Female,Yes,Sat,Dinner,2, ,18.78,3.0,Female,No,Thur,Dinner,2, In [72]: data['perc_tip'].mean() Out[72]: In [73]: # Slow version: #the_sum = 0 #for row in data: # the_sum += row['perc_tip'] # #the_mean = the_sum / len(data) In [74]: data.groupby('size')['perc_tip'].mean() Out[74]: size

11 Name: perc_tip, dtype: float64 In [75]: data.groupby(['size', 'sex'])['perc_tip'].mean() Out[75]: size sex 1 Female Male Female Male Female Male Female Male Female Male Female Male Name: perc_tip, dtype: float64 In [76]: means = data.groupby(['size', 'sex'])['perc_tip'].mean() In [77]: means.unstack('sex') Out[77]: sex Female Male size In [78]: data.groupby(['size', 'sex'])['perc_tip'].mean().unstack().to_latex() Out[78]: '\\begin{tabular}{lrr}\n\\toprule\nsex & Female & Male \\\\\nsize In [79]: # "readable" version: (data.groupby(['size', 'sex']) ['perc_tip'].mean().unstack().to_latex()) Out[79]: '\\begin{tabular}{lrr}\n\\toprule\nsex & Female & Male \\\\\nsize 11

12 2 Slide 3 matplotlib: visualize data In [80]: from matplotlib import pyplot as plt %matplotlib inline In [81]: plt.plot([1,3,2]) Out[81]: [<matplotlib.lines.line2d at 0x7fa9fcc22160>] In [82]: plt.bar([0,1,2], [1,3,2]) Out[82]: <Container object of 3 artists> 12

13 In [83]: data['total_bill'].plot() Out[83]: <matplotlib.axes._subplots.axessubplot at 0x7fa9fcb29320> 13

14 In [84]: data[['total_bill', 'tip']].plot() Out[84]: <matplotlib.axes._subplots.axessubplot at 0x7fa9fcb26dd8> In [85]: data.groupby('size')['perc_tip'].mean().plot(kind='bar') Out[85]: <matplotlib.axes._subplots.axessubplot at 0x7fa9fcb1e128> 14

15 In [86]: data.groupby(['size', 'sex'])['perc_tip'].mean().unstack().plot(kind='bar' Out[86]: <matplotlib.axes._subplots.axessubplot at 0x7fa9fca84e48> In [87]: female = (data['sex'] == 'Female') data.plot(kind='scatter', x='perc_tip', y='total_bill', c=female, edgecolor='r' ) Out[87]: <matplotlib.axes._subplots.axessubplot at 0x7fa9fcb264e0> 15

16 4 Slide 5 Statsmodels In [88]: import statsmodels.api as sm In [89]: res = sm.ols.from_formula('tip ~ total_bill + sex + day + size', data=data In [90]: res.summary() ========================================================================== ========================================================================== Out[90]: <class 'statsmodels.iolib.summary.summary'> """ OLS Regression Results Dep. Variable: tip R-squared: 0 Model: OLS Adj. R-squared: 0 Method: Least Squares F-statistic: 3 Date: Sat, 05 May 2018 Prob (F-statistic): 4.04 Time: 10:36:20 Log-Likelihood: -34 No. Observations: 244 AIC: 7 Df Residuals: 237 BIC: 7 Df Model: 6 Covariance Type: nonrobust 16

17 ========================================================================== ========================================================================== coef std err t P> t [ Intercept sex[t.male] day[t.sat] day[t.sun] day[t.thur] total_bill size Omnibus: Durbin-Watson: 2 Prob(Omnibus): Jarque-Bera (JB): 49 Skew: Prob(JB): 1.87 Kurtosis: Cond. No. Warnings: [1] Standard Errors assume that the covariance matrix of the errors is cor """ In [91]: data['day'].unique() Out[91]: array(['sun', 'Sat', 'Thur', 'Fri'], dtype=object) 6 Slide 7 scikit-learn In [92]: from sklearn.neural_network import MLPClassifier In [93]: clf = MLPClassifier() In [94]: data.head() Out[94]: total_bill tip sex smoker day time size perc_tip Female No Sun Dinner Male No Sun Dinner Male No Sun Dinner Male No Sun Dinner Female No Sun Dinner In [95]: data['sex'] = data['sex'] == 'Female' data['smoker'] = data['smoker'] == 'Yes' data['time'] = data['time'] == 'Dinner' In [96]: data.head() 17

18 Out[96]: total_bill tip sex smoker day time size perc_tip True False Sun True False False Sun True False False Sun True False False Sun True True False Sun True In [97]: data['good_tip'] = data['perc_tip'] > data['perc_tip'].mean() In [98]: x = data.drop(['good_tip', 'day', 'perc_tip', 'tip'], axis=1) y = data['good_tip'] In [99]: data.head() Out[99]: total_bill tip sex smoker day time size perc_tip good_tip True False Sun True False False False Sun True False False False Sun True True False False Sun True False True False Sun True False In [100]: res = clf.fit(x, y) In [101]: # I'M CHEATING! I'M CHEATING! res.score(x, y) Out[101]: In [102]: from sklearn.tree import DecisionTreeClassifier, export_graphviz In [103]: tree = DecisionTreeClassifier(max_depth=4) In [104]: res = tree.fit(x, y) In [105]: res.score(x, y) Out[105]: In [106]: dot_data = export_graphviz(tree, out_file=none, feature_names=x.columns, filled=true, rounded=true, special_characters=true) In [107]: import graphviz graph = graphviz.source(dot_data) In [108]: graph Out[108]: 18

19 True total_bill gini = samples = 169 value = [82, 87] total_bill gini = samples = 244 value = [137, 107] False total_bill 48.3 gini = samples = 75 value = [55, 20] smoker 0.5 gini = samples = 24 value = [6, 18] total_bill gini = samples = 145 value = [76, 69] total_bill gini = samples = 73 value = [55, 18] gini = 0.0 samples = 2 value = [0, 2] time 0.5 gini = samples = 17 value = [6, 11] gini = 0.0 samples = 7 value = [0, 7] total_bill gini = samples = 129 value = [71, 58] total_bill gini = samples = 16 value = [5, 11] size 3.5 gini = samples = 55 value = [38, 17] smoker 0.5 gini = samples = 18 value = [17, 1] gini = samples = 8 value = [1, 7] gini = samples = 9 value = [5, 4] gini = samples = 122 value = [64, 58] gini = 0.0 samples = 7 value = [7, 0] gini = 0.0 samples = 3 value = [0, 3] gini = samples = 13 value = [5, 8] gini = samples = 33 value = [26, 7] gini = samples = 22 value = [12, 10] gini = samples = 9 value = [8, 1] gini = 0.0 samples = 9 value = [9, 0] In [109]: from sklearn.ensemble import RandomForestClassifier In [110]: forest = RandomForestClassifier(max_depth=4) In [111]: res = forest.fit(x, y) In [112]: res.score(x, y) Out[112]: