Repository navigation
Expand file tree
/
Copy pathvisualization_project_code.py
More file actions
431 lines (336 loc) · 17.9 KB
/
Copy pathvisualization_project_code.py
File metadata and controls
431 lines (336 loc) · 17.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
# -*- coding: utf-8 -*-
"""Visualization project code.ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1cOZ8ScqtY5EnA4ozxkvPG4tdckPZvOzL
### 0-Import libraries
"""
from urllib.request import urlopen
import json
import plotly.express as px
import plotly.graph_objects as go
import pandas as pd
from itertools import product
import matplotlib.pyplot as plt
from sklearn import decomposition
import seaborn as sn
import numpy as np
import warnings
warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)
from google.colab import files
data=files.upload()
"""### 1-Load & Pre-process Data"""
# Prepare country data
df_countries = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2014_world_gdp_with_codes.csv')
df_countries.pop('GDP (BILLIONS)')
other_codes = {'COUNTRY': ['USA', 'Bosnia/H.', 'Czech Rep.'], 'CODE':['USA', 'BIH', 'CZE']}
df_other_codes = pd.DataFrame([['USA', 'USA'], ['Bosnia/H.', 'BIH'], ['Czech Rep.', 'CZE']],
columns=['COUNTRY', 'CODE'], index=[222, 223, 224])
df_countries = df_countries.append(df_other_codes)
print((df_countries))
# Load the study data
df = pd.read_table('34658-0001-Data (2).tsv', sep="\t")
df = df[df['SCOUNTRY'] != 'NL Antilles'] # NL Antilles was dissolved in 2010
"""### 2-Geospatial Plot
#### 2.1-Data
"""
df_countries['Cases'] = 0 # Total tested students
df_countries['Alcohol/drug'] = 0 # Total students that got drunk on alcohol or used drugs
df_countries['Crime'] = 0 # Total students that commited delinquency
# df[["SCOUNTRY"]].value_counts()
for i, student in df.iterrows():
country = student["SCOUNTRY"]
alcohol = [student["BEERAGE"], student["SPIRAGE"]] # First time alcohol use
drug = [student["HASHAGE"], student["XTCAGE"], student["LHCAGE"]] # First time drug use
violent = [student["ASLTAGE"], student["GFIGAGE"], student["SNATAGE"], student["VANDAGE"], student["EXTOAGE"]] # First time violent crime
non_violent = [student["BURGAGE"], student["HACKAGE"], student["SHOPAGE"], student["BICTAGE"], student["CARTAGE"]] # First time non-violent crime
relevant_data = list(np.concatenate(list(np.array([alcohol, drug, violent, non_violent]).flat)).flat)
if not any(x in relevant_data for x in [-9, 97, 99]): # Ignore blank/ambigious/no answers
country_index = (df_countries.index[df_countries['COUNTRY'] == country]).tolist()[0]
df_countries.loc[country_index,'Cases'] += 1
if any(x in alcohol for x in [1, 2, 3]) or any(x in drug for x in [1, 2, 3]):
df_countries.loc[country_index,'Alcohol/drug'] += 1
if any(x in violent for x in [1, 2, 3]) or any(x in non_violent for x in [1, 2, 3]):
df_countries.loc[country_index,'Crime'] += 1
# Remove countries with no test data
for i, country in df_countries.iterrows():
count = country["Cases"]
if count == 0:
df_countries = df_countries.drop(i)
df_countries['Alcohol/drug rate (%)'] = 100*df_countries['Alcohol/drug']/df_countries['Cases']
df_countries['Crime rate (%)'] = 100*df_countries['Crime']/df_countries['Cases']
print(df_countries)
"""#### 2.2-Plots
##### Alcohol
"""
fig1 = go.Figure(data=go.Choropleth(
locations = df_countries['CODE'],
z = df_countries['Alcohol/drug rate (%)'],
text = df_countries['COUNTRY'],
colorscale = 'Viridis',
autocolorscale=False,
reversescale=True,
marker_line_color='darkgray',
marker_line_width=0.5,
colorbar_title = 'Alcohol/drug use rate (%)'))
fig1.update_layout(title_text='Alcohol or Drug Use Rate in Students in Grades 7 to 9 By Country', title_x=0.5,
autosize=False, width = 1000, margin=dict(l=0,r=0,b=0,t=30,pad=4,autoexpand=True),
geo=dict(showframe=False, showcoastlines=False, projection_type='mercator'))
fig1.show()
"""##### Delinquency/crime"""
fig2 = go.Figure(data=go.Choropleth(
locations = df_countries['CODE'],
z = df_countries['Crime rate (%)'],
text = df_countries['COUNTRY'],
colorscale = 'Viridis',
autocolorscale=False,
reversescale=True,
marker_line_color='darkgray',
marker_line_width=0.5,
colorbar_title = 'Crime/Delinquency rate (%)'))
fig2.update_layout(title_text='Childhood Delinquency in Students in Grades 7 to 9 By Country', title_x=0.5,
autosize=False, width = 1000, margin=dict(l=0,r=0,b=0,t=30,pad=4,autoexpand=True),
geo=dict(showframe=False, showcoastlines=False, projection_type='mercator'))
fig2.show()
"""### 3-Pie chart
#### 3.1-Data
"""
def get_feature_vect(feature):
feature_vect = np.zeros(len(feature))
for i, e in enumerate(feature):
if e in [1,2,3]:
feature_vect[i] = 1
return feature_vect
Alcohol, Drug, Violent, Non_violent = np.zeros(2), np.zeros(3), np.zeros(5), np.zeros(5)
Clean, No_crime = 0, 0
for i, student in df.iterrows():
country = student["SCOUNTRY"]
alcohol = [student["BEERAGE"], student["SPIRAGE"]] # First time alcohol use
drug = [student["HASHAGE"], student["XTCAGE"], student["LHCAGE"]] # First time drug use
violent = [student["ASLTAGE"], student["GFIGAGE"], student["SNATAGE"], student["VANDAGE"], student["EXTOAGE"]] # First time violent crime
non_violent = [student["BURGAGE"], student["HACKAGE"], student["SHOPAGE"], student["BICTAGE"], student["CARTAGE"]] # First time non-violent crime
relevant_data = list(np.concatenate(list(np.array([alcohol, drug, violent, non_violent]).flat)).flat)
if not any(x in relevant_data for x in [-9, 97, 99]): # Ignore blank/ambigious/no answers
if any(x in alcohol for x in [1, 2, 3]) or any(x in drug for x in [1, 2, 3]):
Alcohol += get_feature_vect(alcohol)
Drug += get_feature_vect(drug)
else:
Clean += 1
if any(x in violent for x in [1, 2, 3]) or any(x in non_violent for x in [1, 2, 3]):
Violent += get_feature_vect(violent)
Non_violent += get_feature_vect(non_violent)
else:
No_crime += 1
alc_drugs = list(np.concatenate(list(np.array([[Clean], Alcohol, Drug]).flat) ).flat)
df_alc_drugs = {'Substance': ['No drugs/alcohol', 'Beer, breezers or wine', 'Strong spirits', 'Weed, marijuana or hash',
'XTC or speed', 'LSD, heroin or coke'],
'Sub_count': alc_drugs}
delinq = list(np.concatenate(list(np.array([[No_crime], Violent, Non_violent]).flat) ).flat)
df_delinq = {'Delinquency': ['No delinquency', 'Assault', 'Group fighting', 'Snatch a purse/bag', 'Vandalism', 'Extortion',
'Burglary', 'Hacking', 'Shoplifting', 'Bicycle theft', 'Car theft'],
'Sub_count': delinq}
print(df_alc_drugs)
print(df_delinq)
"""#### 3.2-Plot
##### Alcohol/drugs
"""
fig3 = px.pie(df_alc_drugs, values='Sub_count', names='Substance')
fig3.update_layout(title_text='Substance Use in Students in Grades 7 to 9', title_x=0.5)
fig3.update_traces(textinfo='percent+label',
marker=dict(colors=px.colors.sequential.Jet, line=dict(color='#000000', width=2)))
fig3.show()
"""##### Delinquency/crime"""
fig4 = px.pie(df_delinq, values='Sub_count', names='Delinquency')
fig4.update_layout(title_text='Childhood Delinquency in Students in Grades 7 to 9', title_x=0.5)
fig4.update_traces(textinfo='percent+label',
marker=dict(colors=px.colors.sequential.Jet, line=dict(color='#000000', width=2)))
fig4.show()
"""### 4-2D Heat Map
#### 4.1-Data
"""
# Most severe delinquency (violent > non-violent > no crime)
# Most severe substance used (LHC > XTC/speed > MJ/hash > spirits > beer/wine)
df_heatmap = pd.DataFrame(columns=['Delinquency', 'Substance'])
sub_order = ['No Drugs/Alcohol', 'Beer, breezers or wine', 'Strong spirits', 'Weed, marijuana or hash',
'XTC or speed', 'LSD, heroin or coke']
for i, student in df.iterrows():
alcohol = [student["BEERAGE"], student["SPIRAGE"]] # First time alcohol use
drug = [student["HASHAGE"], student["XTCAGE"], student["LHCAGE"]] # First time drug use
violent = [student["ASLTAGE"], student["GFIGAGE"], student["SNATAGE"], student["VANDAGE"], student["EXTOAGE"]] # First time violent crime
non_violent = [student["BURGAGE"], student["HACKAGE"], student["SHOPAGE"], student["BICTAGE"], student["CARTAGE"]] # First time non-violent crime
substances = list(np.concatenate(list(np.array([alcohol, drug]).flat)).flat)
relevant_data = list(np.concatenate(list(np.array([alcohol, drug, violent, non_violent]).flat)).flat)
if not any(x in relevant_data for x in [-9, 97, 99]): # Ignore blank/ambigious/no answers
if any(x in alcohol for x in [1, 2, 3]) or any(x in drug for x in [1, 2, 3]):
for j, e in enumerate(substances):
if e in [1,2,3]:
df_heatmap.loc[i,'Substance'] = sub_order[j+1]
else:
df_heatmap.loc[i,'Substance'] = 'No Drugs/Alcohol'
if any(x in non_violent for x in [1, 2, 3]) or any(x in violent for x in [1, 2, 3]):
df_heatmap.loc[i,'Delinquency'] = "Non-violent crimes"
if any(x in violent for x in [1, 2, 3]):
df_heatmap.loc[i,'Delinquency'] = "Violent crimes"
else:
df_heatmap.loc[i,'Delinquency'] = 'No Delinquency'
print(df_heatmap.loc[5:10,'Substance'])
"""#### 4.2-Plot"""
confusion_matrix = pd.crosstab(df_heatmap['Substance'], df_heatmap['Delinquency'], normalize='index')*100
confusion_matrix = confusion_matrix.reindex(sub_order[::-1], axis="rows")
sn.heatmap(confusion_matrix, annot=True, cbar_kws={'label': 'Percentage (%)'}, cmap='coolwarm')
#print(confusion_matrix)
plt.title('Alcohol and Drug Use Corresponding to Delinquency Type (row-wise normalized)')
plt.show()
"""### 5-PCA
#### 5.1-Data
"""
# Most severe delinquency (violent > non-violent > no crime)
df_pca = pd.DataFrame(columns=['Delinquency', 'Alcohol Score', 'Drug Score', 'NHOODBO', 'NHOODDO', 'NHOODIN', 'NHOOD',
'FAMAFF', 'FAMBOND', 'FAMWORK', 'SCHOOLCL', 'SCHOOLDO'])
social_vars = ['NHOODBO', 'NHOODDO', 'NHOODIN', 'NHOOD', 'FAMAFF', 'FAMBOND', 'FAMWORK', 'SCHOOLCL', 'SCHOOLDO']
print(df.shape)
for i, student in df.iterrows():
alcohol, drug = [student["BEERLMC"], student["SPIRLMC"]], [student["HASHLMC"], student["XTCLMC"], student["LHCLMC"]]
violent = [student["ASLTAGE"], student["GFIGAGE"], student["SNATAGE"], student["VANDAGE"], student["EXTOAGE"]] # First time violent crime
non_violent = [student["BURGAGE"], student["HACKAGE"], student["SHOPAGE"], student["BICTAGE"], student["CARTAGE"]] # First time non-violent crime
nhood = [student["NHOODBO"], student["NHOODDO"], student["NHOODIN"], student["NHOOD"]]
fam_sch = [student["FAMAFF"], student["FAMBOND"], student["FAMWORK"], student["SCHOOLCL"], student["SCHOOLDO"]]
substances = list(np.concatenate(list(np.array([alcohol, drug]).flat)).flat)
crimes = list(np.concatenate((np.array([violent, non_violent]))).flat)
socials = list(np.concatenate(list(np.array([nhood, fam_sch]).flat)).flat)
data_valid_alcdrug = not((7 in substances) or (9 in substances))
data_valid_crime = not((-9 in crimes) or (97 in crimes) or (99 in crimes))
data_valid_social = not((9 == student["FAMWORK"]) or (any(x < 0 for x in socials)))
if data_valid_alcdrug and data_valid_crime and data_valid_social: # Ignore blank/ambigious/no answers
df_pca.loc[i,'Alcohol Score'] = np.sum(np.array(alcohol))
df_pca.loc[i,'Drug Score'] = np.sum(np.array(drug))
if any(x in non_violent for x in [1, 2, 3]) or any(x in violent for x in [1, 2, 3]):
df_pca.loc[i,'Delinquency'] = 1 # Non-violent crimes
if any(x in violent for x in [1, 2, 3]):
df_pca.loc[i,'Delinquency'] = 2 # Violent crimes
else:
df_pca.loc[i,'Delinquency'] = 0 # No Delinquency
for var in social_vars:
df_pca.loc[i, var] = student[var]
print(df_pca.shape)
print(df_pca.loc[5:10])
"""#### 5.2-Plot"""
np.random.seed(127)
X = df_pca[df_pca.columns.difference(['Delinquency'])].to_numpy()
y = df_pca.loc[:, df_pca.columns == 'Delinquency'].to_numpy().T[0]
pca = decomposition.PCA(n_components=3)
pca.fit(X)
X = pca.transform(X)
print("Explained variance of the axes: ", pca.explained_variance_ratio_)
print("Importance given to the features for each principal axis (10 being the most important):")
for axis in pca.components_:
print(axis.argsort().argsort())
# Undersample for plotting
sample_size = 100
mask = np.array([True] * sample_size + [False] * (len(y) - sample_size))
np.random.shuffle(mask)
X_new, y_new, w = [], [], [6, 1, 2]
w0, w1, w2 = [5], [2], [1]
for i, val in enumerate(y[mask]):
random = round(np.random.randint(w[val]))
if random == w[val]-1:
X_new.append(X[mask][i])
y_new.append(y[mask][i])
X, y = np.array(X_new), np.array(y_new)
# Plotting sample distribution
# plt.figure(0)
# n, bins, patches = plt.hist(y, facecolor='g', alpha=0.75)
# plt.grid(True)
fig5 = plt.figure(1)
plt.clf()
ax = fig5.add_subplot(111, projection="3d", elev=22, azim=65)
ax.set_position([0, 0, 0.95, 1])
plt.cla()
pca = decomposition.PCA(n_components=3)
pca.fit(X)
X = pca.transform(X)
colors = ['b', 'y', 'r']
for name, label in [("No Delinquency", 0), ("Non-violent crimes", 1), ("Violent crimes", 2)]:
ax.scatter(X[y == label, 0], X[y == label, 1], X[y == label, 2], color=colors[label], edgecolor="k", label=name)
ax.set_xlabel('1st Principal Component')
ax.set_ylabel('2nd Principal Component')
ax.set_zlabel('3rd Principal Component')
ax.legend()
plt.show()
data_pca = np.c_[X, y]
names = []
for label in y:
if label==0:
names.append('No Delinquency')
elif label == 1:
names.append('Non-violent crimes')
elif label == 2:
names.append('Violent crimes')
data_pca = np.c_[data_pca, np.array(names)]
df_pca2 = pd.DataFrame(data_pca, columns = ['PCA1','PCA2','PCA3','Label','Name'])
df_pca2['PCA1'], df_pca2['PCA2'], df_pca2['PCA3'] = df_pca2['PCA1'].astype(float), df_pca2['PCA2'].astype(float), df_pca2['PCA3'].astype(float)
#print(df_pca2)
fig6 = px.scatter_3d(df_pca2, x='PCA1', y='PCA2', z='PCA3', color='Name')
fig6.show()
"""### 6-Scatter Plot
#### 6.1-Data
"""
# Most severe delinquency (violent > non-violent > no crime)
df_scatter = pd.DataFrame(columns=['Alcohol Score', 'Drug Score', 'Violent Score', 'Non-Violent Score', 'Delinquency Score', 'Substance Score'])
print(df.shape)
for i, student in df.iterrows():
alcohol, drug, var_str1 = [], [], ["LTP", "LMC"]
for j in range(len(var_str1)):
alcohol.append(student["BEER"+var_str1[j]])
alcohol.append(student["SPIR"+var_str1[j]])
drug.append(student["HASH"+var_str1[j]])
drug.append(student["XTC"+var_str1[j]])
drug.append(student["LHC"+var_str1[j]])
violent, non_violent, var_str2 = [], [], ["LTP", "LYC"]
for j in range(len(var_str2)):
violent.append(student["ASLT"+var_str2[j]])
violent.append(student["GFIG"+var_str2[j]])
violent.append(student["SNAT"+var_str2[j]])
violent.append(student["VAND"+var_str2[j]])
violent.append(student["EXTO"+var_str2[j]])
non_violent.append(student["BURG"+var_str2[j]])
non_violent.append(student["HACK"+var_str2[j]])
non_violent.append(student["SHOP"+var_str2[j]])
non_violent.append(student["BICT"+var_str2[j]])
non_violent.append(student["CART"+var_str2[j]])
substances = list(np.concatenate(list(np.array([alcohol, drug]).flat)).flat)
crimes = list(np.concatenate((np.array([violent, non_violent]))).flat)
data_valid_alcdrug = not((7 in substances) or (9 in substances))
data_valid_crime = not((7 in crimes) or (9 in crimes))
if data_valid_alcdrug and data_valid_crime: # Ignore blank/ambigious/no answers
len_alcs = round(len(alcohol)/len(var_str1))
alc_arr = np.array(alcohol)
df_scatter.loc[i,'Alcohol Score'] = np.sum(alc_arr[0:len_alcs])+np.sum(alc_arr[len_alcs:])*0.25
# scale last month classes from [0, 4] to [0, 1]
len_drgs = round(len(drug)/len(var_str1))
drg_arr = np.array(drug)
df_scatter.loc[i,'Drug Score'] = np.sum(drg_arr[0:len_drgs])+np.sum(drg_arr[len_drgs:])*0.25
len_viol = round(len(violent)/len(var_str2))
viol_arr = np.array(violent)
df_scatter.loc[i,'Violent Score'] = np.sum(viol_arr[0:len_viol])+np.sum(viol_arr[len_viol:])*0.25
len_nonviol = round(len(non_violent)/len(var_str2))
nonviol_arr = np.array(non_violent)
df_scatter.loc[i,'Non-Violent Score'] = np.sum(nonviol_arr[0:len_nonviol])+np.sum(nonviol_arr[len_nonviol:])*0.25
df_scatter['Delinquency Score'] = df_scatter['Violent Score'] + df_scatter['Non-Violent Score']
df_scatter['Substance Score'] = df_scatter['Drug Score'] + df_scatter['Alcohol Score']
df_scatter = df_scatter.astype(float)
print(df_scatter.shape)
print(df_scatter.loc[5:10])
"""#### 6.2-Plot"""
df_scatter2 = pd.DataFrame(columns=['Alcohol Score', 'Drug Score', 'n', 'Avg. Delinquency Score'])
l1, l2 = df_scatter['Alcohol Score'].unique(), df_scatter['Drug Score'].unique()
df_combs = pd.DataFrame(list(product(l1, l2)), columns=['Alcohol Score', 'Drug Score'])
df_scatter2['Alcohol Score'], df_scatter2['Drug Score'] = df_combs['Alcohol Score'], df_combs['Drug Score']
for i, combination in df_scatter2.iterrows():
alc, drug = combination['Alcohol Score'], combination['Drug Score']
df_filtered = df_scatter[(df_scatter['Alcohol Score'] == alc) & (df_scatter['Drug Score'] == drug)]
df_scatter2.loc[i,'Avg. Delinquency Score'] = df_filtered["Delinquency Score"].mean()
df_scatter2.loc[i,'n'] = df_filtered.shape[0]
df_scatter2 = df_scatter2.astype(float) # Use only instances of where more than 10 samples exist for a combination
fig_scatter = px.scatter(df_scatter2[df_scatter2['n'] > 10], x="Alcohol Score", y="Drug Score", size='Avg. Delinquency Score', color='Avg. Delinquency Score', color_continuous_scale=px.colors.sequential.Inferno)
fig_scatter.show()