Consulting - Quality
Difference makes the DIFFERENCE
Agriculture Dataset
Practice - Cleaning dataset
In [ ]:
import numpy as np
import pandas as pd
In [ ]:
df = pd.read_csv('/content/agriculture_ds.csv')
In [ ]:
df.head()
Out[ ]:
display only null values in any column in a dataframe¶
In [ ]:
df[df.isna().any(axis=1)]
Out[ ]:
Display if state_name has any null values¶
In [ ]:
df[df['State_Name'].isnull()]
Out[ ]:
In [ ]:
df[df['District_Name'].isnull()]
Out[ ]:
In [ ]:
df[df['Crop_Year'].isnull()]
Out[ ]:
Display, a column for a particular value in that column¶
In [ ]:
df[df['State_Name'] == "Andhra Pradesh"]
Out[ ]:
In [ ]:
df[df['State_Name'] == "Andhra Pradesh"].count()
Out[ ]:
apply condition on two different columns in dataframe¶
In [ ]:
df[(df['State_Name'] == "Bihar") & (df['Season'] == "Kharif")]
Out[ ]:
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") & (df['Crop'] == 'Bajra')]
Out[ ]:
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") | (df['Season'] == 'Kharif')]
Out[ ]:
though we have Rabi season for the selected state, it doesnt display because the actual lenght of Rabi is 4 chars but in dataframe it is 11 chars¶
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") & (df['Season'] == "Rabi")]
Out[ ]:
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") & (df['Season'] == "Kharif")]
Out[ ]:
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") & (df['Season'] == "Whole Year")]
Out[ ]:
Strip or remove all traling spaces from a string in a particular column¶
In [ ]:
df['Season'] = df['Season'].str.strip()
In [ ]:
df['Season Len'] = df['Season'].str.len()
In [ ]:
df.head()
Out[ ]:
now the condition works - after stripping the trailing spaces in the Season Column¶
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") & (df['Season'] == "Kharif")]
Out[ ]:
In [ ]:
df[(df['State_Name'] == "Andhra Pradesh") & (df['Season'] == "Rabi")]
Out[ ]:
In [ ]:
df[['State_Name', 'Season', 'Season Len']]
Out[ ]:
In [ ]:
df.drop('Season Len', axis=1, inplace=True)
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df['Crop Len'] = df['Crop'].str.len()
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df['State_Name len'] = df['State_Name'].str.len()
In [ ]:
df.head(3)
Out[ ]:
In [ ]:
df[['State_Name', 'State_Name len']]
Out[ ]:
In [ ]:
df['Dist Len'] = df['District_Name'].str.len()
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df[['District_Name', 'Dist Len']]
Out[ ]:
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df.drop(['State_Name len', 'Dist Len', 'Crop Len'], inplace = True, axis = 1)
now, the length of each data value in each column is equal to the value space - extra traling spaces are removed¶
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df[df['Production'] == '=']
Out[ ]:
In [ ]:
df.sort_values('Production', inplace=True, ascending=False)
In [ ]:
df.head()
Out[ ]:
In [ ]:
df[df['Production'] == "="]
Out[ ]:
In [ ]:
# index_names = df[ df['Production'] == 21 ].index
index_names = df[ df['Production'] == "=" ].index
df.drop(index_names, inplace = True)
In [ ]:
df[df['Production'] == "="]
Out[ ]:
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df
Out[ ]:
In [ ]:
df.dropna(inplace = True)
In [ ]:
df
Out[ ]:
convert Crop_year col to string so thta it may not sum up on group functions¶
In [ ]:
df['Crop_Year'] = df['Crop_Year'].astype(str)
In [ ]:
df.head(2)
Out[ ]:
In [ ]:
df.dtypes
Out[ ]:
convert production col to float datatype¶
In [ ]:
df['Production'] = pd.to_numeric(df['Production'])
In [ ]:
df.dtypes
Out[ ]:
In [ ]:
df.to_csv('/content/clean_agri_ds.csv')
In [ ]:
df.describe()
Out[ ]:
In [ ]:
df.head(3)
Out[ ]:
End Cell¶
In [ ]: