Consulting - Quality
Difference makes the DIFFERENCE
Fundamentals of DataScience Week 10 - Class Demo
In [1]:
import numpy as np
import pandas as pd
Creating dataframe objects¶
In [2]:
arr = np.random.randint(0, 10, (5, 3))
In [3]:
arr
Out[3]:
In [4]:
df = pd.DataFrame(arr)
In [5]:
df
Out[5]:
In [6]:
df.values
Out[6]:
In [7]:
df.index
Out[7]:
In [8]:
df.columns
Out[8]:
In [9]:
for c in df.columns:
print(c)
In [10]:
df.values[0]
Out[10]:
In [11]:
df.index = ['R1', 'R2', 'R3', 'R4', 'R5']
df.columns = ['C1', 'C2', 'C3']
In [12]:
df
Out[12]:
In [13]:
df.loc['R3', 'C2']
Out[13]:
In [14]:
df.iloc[2, 1]
Out[14]:
In [15]:
type(df.iloc[2:4, 1:3])
Out[15]:
In [16]:
df.loc['R3':'R5', 'C2':'C3']
Out[16]:
In [17]:
df.iloc[0]
Out[17]:
In [18]:
df.iloc[:, 0]
Out[18]:
In [25]:
df.shape
Out[25]:
In [26]:
df.T
Out[26]:
Task on creating dataframes
In [27]:
def create_df(nRows, nCols, maxRand=10):
arr = np.random.randint(0, maxRand, (nRows, nCols))
df = pd.DataFrame(arr)
df.index = ['R' + str(x) for x in np.arange(1, nRows+1)]
df.columns = ['C' + str(x) for x in np.arange(1, nCols+1)]
return df
In [28]:
create_df(5, 3)
Out[28]:
In [29]:
create_df(2, 5)
Out[29]:
In [30]:
mass = pd.Series([0.33, 4.87, 5.97, 0.642, 1898, 568, 86.8, 102, 0.0146],
index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
diameter = pd.Series([4879, 12104, 12756, 3475, 6792, 142984, 120536, 51118, 49528, 2370],
index=['Mercury', 'Venus', 'Earth', 'Moon', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
In [31]:
mass
Out[31]:
In [32]:
diameter
Out[32]:
In [33]:
df = pd.DataFrame({'Mass': mass, 'Diameter': diameter})
In [34]:
df
Out[34]:
In [35]:
df['Mass']
Out[35]:
In [36]:
df['Diameter']
Out[36]:
In [37]:
df['Earth']
In [45]:
df['Mass']['Earth']
Out[45]:
In [46]:
df.Mass.Earth
Out[46]:
In [51]:
df['pop'] = 0
In [53]:
df
Out[53]:
In [54]:
df['pop']['Earth'] = 8000000000
In [55]:
df
Out[55]:
In [56]:
df['Mass'] is df.Mass
Out[56]:
In [57]:
df['pop'] is df.pop
Out[57]:
In [58]:
df
Out[58]:
In [59]:
df
Out[59]:
In [60]:
df.loc['Earth']
Out[60]:
In [61]:
df.loc[:,'Mass']
Out[61]:
In [78]:
def create_mean_row(df):
# df.loc['Col_Mean'] = [np.mean(df[col]) for col in df.columns]
df.loc['Col_Mean'] = df.mean()
return df
In [62]:
df
Out[62]:
In [63]:
df.loc['Col_Mean'] = 0
In [64]:
df
Out[64]:
In [65]:
np.mean(df['Mass'])
Out[65]:
In [69]:
df.drop('Col_Mean', inplace=True)
In [70]:
df.drop('pop', axis=1, inplace=True)
In [71]:
df
Out[71]:
In [72]:
np.mean(df['Mass'])
Out[72]:
In [73]:
df.loc['Col_Mean'] = [np.mean(df['Mass']), np.mean(df['Diameter'])]
In [74]:
df
Out[74]:
In [76]:
df.drop('Col_Mean')
Out[76]:
In [79]:
create_mean_row(df)
Out[79]:
In [80]:
df = create_df(5, 3)
In [82]:
df
Out[82]:
In [81]:
df.mean()
Out[81]:
In [83]:
df.mean(axis=1)
Out[83]:
In [85]:
df['Row_Mean'] = df.mean(axis=1)
In [88]:
df.loc['Col_Mean'] = df.mean()
In [89]:
df
Out[89]:
In [90]:
df.median()
Out[90]:
In [91]:
df.min()
Out[91]:
In [92]:
df.max()
Out[92]:
In [93]:
df.quantile(0.25)
Out[93]:
In [94]:
df.drop('Row_Mean', axis=1)
Out[94]:
In [95]:
df.describe()
Out[95]:
In [96]:
mass = pd.Series([0.33, 4.87, 5.97, 0.642, 1898, 568, 86.8, 102, 0.0146],
index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
diameter = pd.Series([4879, 12104, 12756, 3475, 6792, 142984, 120536, 51118, 49528, 2370],
index=['Mercury', 'Venus', 'Earth', 'Moon', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
In [97]:
planets = pd.DataFrame({'mass': mass, 'diameter': diameter})
In [98]:
planets.describe()
Out[98]:
In [99]:
import seaborn as sns
In [139]:
df = sns.load_dataset('planets')
In [102]:
df.info()
In [103]:
df.head()
Out[103]:
In [104]:
df.tail()
Out[104]:
In [105]:
df.describe()
Out[105]:
Go through each row of the dataframe and delete it (drop) if any of the columns is null
In [106]:
for r in df.index:
for c in df.columns:
if pd.isnull(df.loc[r, c]):
df.drop(r, inplace=True)
break
In [107]:
df.describe()
Out[107]:
In [110]:
for i, r in df.iterrows():
print(i)
print(r)
break
In [115]:
for i, r in df.iterrows():
if pd.isnull(r).any():
df.drop(i, inplace=True)
In [118]:
df.describe()
Out[118]:
In [119]:
df.dropna(inplace=True)
In [120]:
df.describe()
Out[120]:
Filter and show only those rows which have planets that are found in the 2010s and method is 'Radial Velocity' or 'Transit' and distance is large (> 75 percentile)
In [130]:
df_ = df.copy()
per_75 = df.distance.quantile(0.75)
for i, r in df_.iterrows():
if r['year'] < 2010:
df_.drop(i, inplace=True)
continue
if r['method'] != 'Radial Velocity' and r['method'] != 'Transit':
df_.drop(i, inplace=True)
continue
if r['distance'] < per_75:
df_.drop(i, inplace=True)
continue
In [131]:
df_.describe()
Out[131]:
In [134]:
df_.tail()
Out[134]:
In [133]:
per_75
Out[133]:
In [136]:
df_ = df.copy()
df_ = df_[
(df_['year'] >= 2010) &
((df_['method'] == 'Radial Velocity') | (df_['method'] == 'Transit')) &
(df_['distance'] > per_75)
]
In [137]:
df_.describe()
Out[137]:
Modify the method column to have only the abbreviation of each method
In [140]:
df.method.unique()
Out[140]:
In [145]:
s = 'Orbital Brightness Modulation'
In [146]:
''.join([x[0] for x in s.split(' ')])
Out[146]:
In [147]:
short_names = {}
for s in df.method.unique():
short_names[s] = ''.join([x[0] for x in s.split(' ')])
In [148]:
print(short_names)
In [149]:
for i, r in df.iterrows():
df.loc[i, 'short_method'] = short_names.get(r['method'], r['method'])
In [151]:
df.tail()
Out[151]:
In [152]:
df = sns.load_dataset('planets')
In [153]:
def shorten_method(s):
return short_names.get(s, s)
In [154]:
df['short_method'] = df['method'].apply(shorten_method)
In [155]:
df.head()
Out[155]:
Count the number of planets discovered for each method type
- Split the dataframe into smaller chunks (in this case they should have the same method name)
- Apply some function in each smaller chunk (in this case it is the count function)
- Aggregate the results from each chunk together
In [160]:
d = {}
for m in df.method.unique():
d[m] = df[df.method == m]['method'].count()
print(d)
In [162]:
df.groupby('method')['method'].count()
Out[162]:
In [163]:
d = {}
for m in df.method.unique():
d[m] = df[df.method == m]['distance'].mean()
print(d)
In [164]:
df.groupby('method')['distance'].mean()
Out[164]:
Find out what fraction of planets have been found in the last decade (i.e., in 2010s) across each method type
- Filter the data for given condition (in this case planet found in last decade)
- Split (in this case across method)
- Apply (in this case just count)
- Aggregate (to represent the final result)
In [170]:
s_2010s = df[df.year >= 2010].groupby('method')['method'].count()
In [171]:
s_allTime = df.groupby('method')['method'].count()
In [172]:
s_2010s/s_allTime
Out[172]:
Find a dataset of Nifty numbers for 2018 and 2019 - daily numbers open, close, high, low
In [173]:
nifty50_2018 = pd.read_csv('NIFTY50_2018.csv')
In [174]:
nifty50_2018.head()
Out[174]:
In [175]:
nifty50_2018 = pd.read_csv('NIFTY50_2018.csv', index_col=0)
In [176]:
nifty50_2018.head()
Out[176]:
In [177]:
nifty50_2018.loc['31 Dec 2018']
Out[177]:
In [178]:
nifty50_2018['Open']
Out[178]:
In [179]:
nifty50_2019 = pd.read_csv('NIFTY50_2019.csv', index_col=0)
In [180]:
nifty50_2019.head()
Out[180]:
In [182]:
print(nifty50_2018.shape, nifty50_2019.shape)
In [183]:
nifty50 = pd.concat([nifty50_2018, nifty50_2019])
In [184]:
nifty50.shape
Out[184]:
In [185]:
niftynext50_2019 = pd.read_csv('NIFTYNext50_2019.csv', index_col = 0)
In [186]:
niftynext50_2019.head()
Out[186]:
In [187]:
nifty_2019 = pd.concat([nifty50_2019, niftynext50_2019], axis=1)
In [188]:
nifty_2019.shape
Out[188]:
In [189]:
nifty_2019.head()
Out[189]:
In [190]:
nifty_2019 = pd.concat([nifty50_2019, niftynext50_2019], axis=1,
keys=['nifty50', 'niftynext50'])
In [191]:
nifty_2019.shape
Out[191]:
In [192]:
nifty_2019.head()
Out[192]:
In [196]:
nifty_2019['nifty50'].loc['31 Dec 2019']
Out[196]:
Tasks on the NIFTY datasets:
In 2019, in how many days was the NIFTY50 volatile (high > 105% of low)
In 2019, in how many days was the NIFTYNEXT50 volatile (high > 105% of low)
In 2019, how many days belonged to the four classes NIFTY50 volatile / non-volatile and NIFTYNext50 volatile / non-volatile
Compute the mean, median, std, var of closing values for each weekday in NIFTY50 for 2019
Compute the mean, median, std, var of closing values for each month in NIFTY50 for 2019
On the days in which NIFTY50 closed higher than the open, what was the mean of (close - open) for NIFTYNext50
In 2019, how many days had the day's high lower than the previous day's low in NIFTY50
In 2019, on how many days did the day's close exceed the 30 day moving average in NIFTY50 (exclude first month)
In [ ]:
In [ ]: