Consulting - Quality
Difference makes the DIFFERENCE
PANDAS DATAFRAME
class work
In [ ]:
import numpy as np
import pandas as pd
mass = pd.Series([0.33, 4.87, 5.97, 0.642, 1898, 568, 86.8, 102, 0.0146], index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
diameter = pd.Series([4879, 120104, 12756, 6792, 142984, 120536, 51118, 49528, 2376], index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
- DataFrame is nothing but a collection of series either row wise or col wise or both
- it can be created from numpy array as well
DataFrame in pandas¶
Creating a two dimensional numpy array¶
In [ ]:
arr = np.random.randint(1, 30, (5, 3))
arr
Out[ ]:
In [ ]:
arr
Out[ ]:
In [ ]:
# Creating a dataframe from the numpy array
df = pd.DataFrame(arr)
In [ ]:
df
Out[ ]:
In [ ]:
df.values
Out[ ]:
In [ ]:
df.columns
Out[ ]:
In [ ]:
df[2:3]
Out[ ]:
In [ ]:
df.index
Out[ ]:
In [ ]:
for c in df.columns:
print(c)
In [ ]:
df.index = ["R1", "R2", "R3", "R4", "R5"]
df.columns = ["C1", "C2", "C3"]
In [ ]:
df
Out[ ]:
Using Loc and iLoc functions¶
returns a single element¶
In [ ]:
df.loc['R3', 'C2']
Out[ ]:
In [ ]:
df.iloc[2,1]
Out[ ]:
In [ ]:
df.iloc[4,2]
Out[ ]:
returns a DataFrame¶
In [ ]:
df.iloc[2:4, 1:3]
Out[ ]:
In [ ]:
type(df.iloc[2:4, 1:3])
Out[ ]:
returns a series¶
In [ ]:
arr
Out[ ]:
In [ ]:
df.iloc[0]
Out[ ]:
In [ ]:
type(df.iloc[0])
Out[ ]:
In [ ]:
df.iloc[:2]
Out[ ]:
In [ ]:
df.iloc[:1]
Out[ ]:
In [ ]:
df.iloc[0:3]
Out[ ]:
In [ ]:
df.iloc[:3]
Out[ ]:
In [ ]:
df.iloc[:,0]
Out[ ]:
In [ ]:
df.iloc[:,2]
Out[ ]:
In [ ]:
df.iloc[2,:]
Out[ ]:
In [ ]:
arr
Out[ ]:
In [ ]:
df.loc["R2"]
Out[ ]:
In [ ]:
df.shape
Out[ ]:
In [ ]:
df.T
Out[ ]:
homework to create a dataframe¶
In [ ]:
def create_df(nrows, ncols):
arr1 = np.random.randint(1, 30, (nrows, ncols))
df = pd.DataFrame(arr1)
print(arr1)
In [ ]:
%%time
create_df(3, 4)
In [ ]:
create_df(4, 5)
In [ ]:
def createDF(nRows, nCols):
df = pd.DataFrame(np.random.randint(1, 30, (nRows, nCols)))
print(df)
In [ ]:
%%time
createDF(4, 3)
In [ ]:
createDF(10, 10)
as taught in class¶
In [ ]:
import numpy as np
import pandas as pd
In [ ]:
def create_DF(nRows, nCols, maxData=10):
arr = np.random.randint(0, maxData, (nRows, nCols))
df = pd.DataFrame(arr)
df.index = ["R" + str(x) for x in np.arange(1, nRows + 1)]
df.columns = ["C" + str(x) for x in np.arange(1, nCols + 1)]
return df
In [ ]:
create_DF(4, 5, 50)
Out[ ]:
In [ ]:
create_DF(2, 5)
Out[ ]:
to create DataFrame by invoking multiple series objects¶
In [ ]:
mass = pd.Series([0.33, 4.87, 5.97, 0.642, 1898, 568, 86.8, 102, 0.0146],
index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
diameter = pd.Series([4879, 120104, 12756, 6792, 142984, 120536, 51118, 49528, 2376],
index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
In [ ]:
mass
Out[ ]:
In [ ]:
diameter
Out[ ]:
In [ ]:
df = pd.DataFrame({'mass': mass, 'diameter':diameter})
In [ ]:
df
Out[ ]:
adding a new element to Series Object¶
In [ ]:
diameter = diameter.append(pd.Series(3475, index=['Moon']))
In [ ]:
diameter
Out[ ]:
In [ ]:
df_moon = pd.DataFrame({'mass': mass, 'diameter':diameter})
because DataFrame couldnot identify the item "Moon" in the first dataset, it is placed as NaN (Not a Number)¶
In [ ]:
df_moon
Out[ ]:
In [ ]:
df['mass']
Out[ ]:
In [ ]:
df['diameter']
Out[ ]:
In [ ]:
df = pd.DataFrame({'mass': mass, 'diameter':diameter})
the following data is retrieved from the recent dataframe and nothing to do with Series¶
In [ ]:
df['mass']
Out[ ]:
In [ ]:
df['diameter']
Out[ ]:
In [ ]:
df['mass']['Earth']
Out[ ]:
In [ ]:
df.mass.Earth
Out[ ]:
In [ ]:
df['diameter']['Earth']
Out[ ]:
In [ ]:
df.diameter.Earth
Out[ ]:
adding a new column to an existing dataframe¶
In [ ]:
df['Population'] = 0
In [ ]:
df
Out[ ]:
In [ ]:
df['Population']['Earth'] = 8000000000
In [ ]:
df
Out[ ]:
In [ ]:
df['pop'] = 0
In [ ]:
df
Out[ ]:
In [ ]:
df['mass'] == df.mass
Out[ ]:
In [ ]:
df.mass is df['mass']
Out[ ]:
method to access elements as series¶
In [ ]:
# returns series - horizontal
df.loc['Earth']
Out[ ]:
In [ ]:
type(df.loc['Earth'])
Out[ ]:
In [ ]:
# returns series - Vertical
df.loc[:, 'mass']
Out[ ]:
In [ ]:
type(df.loc[:, 'mass'])
Out[ ]:
the reason for FALSE as result is that pop is a built-in function, hence built-in f() takes precedence¶
- this is one of the pitfalls with using . (Dot) notation hence it is advised to use [] notation
In [ ]:
df['pop'] is df.pop
Out[ ]:
Homework¶
- how to create a new row
- create a mew f() to create a new row which would be a MEAN to calculate the mean of the columns mass and diameter
In [ ]:
df2 = df
In [ ]:
df2
Out[ ]:
In [ ]:
new_row = {'mass':1000, 'diameter':50000,
'Population':0, 'pop':12345678}
df2.loc['New Row'] = new_row
In [ ]:
df2
Out[ ]:
In [ ]:
Mean_row = {'mass': np.mean(mass), 'diameter': np.mean(diameter)}
df2.loc['Mean_row'] = Mean_row
In [ ]:
df2
Out[ ]:
as taught in class¶
In [ ]:
df.loc['Col_Mean'] = 0
In [ ]:
df
Out[ ]:
In [ ]:
np.mean(df['mass'])
Out[ ]:
to delete a row or a column¶
In [ ]:
df.drop('Col_Mean')
Out[ ]:
In [ ]:
df.drop('pop', axis=1)
Out[ ]:
to reflect the changes (deletion) in the original dataframe use INPLACE¶
In [ ]:
df
Out[ ]:
In [ ]:
df.drop('pop', axis = 1, inplace=True)
In [ ]:
df
Out[ ]:
In [ ]:
df.drop('Col_Mean', inplace = True)
In [ ]:
df
Out[ ]:
In [ ]:
df.drop('Population', axis = 1, inplace = True)
In [ ]:
df
Out[ ]:
In [ ]:
df.drop('Mean_row', inplace = True)
In [ ]:
df.drop('New Row', inplace = True)
In [ ]:
df
Out[ ]:
In [ ]:
def create_mean_row(df):
df.loc['Col_Mean'] = [np.mean(df[col]) for col in df.columns]
return df
In [ ]:
create_mean_row(df)
Out[ ]:
In [ ]:
def create_New_Mean_Row(df):
df.loc['Col_Mean_row'] = df.mean()
return df
In [ ]:
create_New_Mean_Row(df)
Out[ ]:
In [ ]:
dff = pd.DataFrame(arr)
In [ ]:
dff.mean()
Out[ ]:
In [ ]:
dff.mean(axis = 1)
Out[ ]:
In [ ]:
dff
Out[ ]:
In [ ]:
dff['row_mean'] = dff.mean(axis = 1)
In [ ]:
dff
Out[ ]:
In [ ]:
dff
Out[ ]:
In [ ]:
dff.loc['col_mean'] = dff.mean()
In [ ]:
dff
Out[ ]:
In [ ]:
df.median()
Out[ ]:
In [ ]:
dff.loc['col_median'] = dff.median()
In [ ]:
dff
Out[ ]:
In [ ]:
dff['row_median'] = dff.median(axis = 1)
In [ ]:
dff
Out[ ]:
In [ ]:
dff['row_stddev'] = df.std(axis=1)
In [ ]:
dff
Out[ ]:
In [ ]:
dff.loc['col_stddev'] = dff.std()
In [ ]:
dff
Out[ ]:
In [ ]:
dff.quantile(0.25)
Out[ ]:
In [ ]:
dff.min()
Out[ ]:
In [ ]:
dff.max()
Out[ ]:
In [ ]:
dff.describe()
Out[ ]:
In [ ]:
df
Out[ ]:
In [ ]:
mass = pd.Series(
[0.33, 4.87, 5.97, 0.642, 1898, 568, 86.8, 102, 0.0146],
index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
diameter = pd.Series(
[4879, 120104, 12756, 6792, 142984, 120536, 51118, 49528, 2376],
index=['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune', 'Pluto'])
In [ ]:
planets = pd.DataFrame({'mass': mass, 'diameter':diameter})
In [ ]:
planets.describe()
Out[ ]:
In [ ]:
planets.describe
Out[ ]:
In [1]:
!pip install nbconvert
%shell jupyter nbconvert --to html /content/testfile.ipynb
In [ ]: