Consulting - Quality
Difference makes the DIFFERENCE
Agriculture OpenSource DataSet
Visualisation with Plotly Express
In [ ]:
! pip3 install plotly==5.7.0
! pip3 install chart_studio
In [2]:
import pandas as pd
import numpy as np
import seaborn as sns
import plotly
import plotly.io as pio
In [3]:
import cufflinks as cf
from plotly.offline import download_plotlyjs, iplot, plot, init_notebook_mode
init_notebook_mode(connected=True)
cf.go_offline()
In [4]:
# ~~~~~~~~~~~~~~~~~~ Code for Plotly and CuffLinks ~~~~~~~~~~~~~~~~~~~~~
# reference: https://www.repath.in/Plotly-and-Cuff-Links/
#~~~~~~~~~~~~~~~~ the following function helps Google Colab to display graphs
# and should be used in every cell, where ever a graph needs to be plotted ~~~~~~~~~~~~~~~~~~~
def configure_plotly_browser_state():
import IPython
display(IPython.core.display.HTML('''
<script src="/static/components/requirejs/require.js"></script>
<script>
requirejs.config({
paths: {
base: '/static/base',
plotly: 'https://cdn.plot.ly/plotly-latest.min.js?noext',
},
});
</script>
'''))
# configure_plotly_browser_state()
# df.plot()
In [6]:
df = pd.read_csv('/content/agriculture_ds.csv')
In [7]:
df.Production.isnull().sum()
Out[7]:
In [ ]:
df.shape()
In [8]:
df.shape
Out[8]:
In [9]:
df.head()
Out[9]:
In [10]:
sns.heatmap(df.isnull(), cmap = 'Blues_r')
Out[10]:
In [12]:
df.head(1)
Out[12]:
In [13]:
df.State_Name.unique()
Out[13]:
In [ ]:
df.Crop_Year.unique()
Out[ ]:
In [ ]:
df.Season.unique()
Out[ ]:
In [ ]:
df.Crop.unique()
Out[ ]:
In [14]:
df.dtypes
Out[14]:
In [15]:
df.info()
In [16]:
df = pd.read_csv('/content/agriculture_ds.csv', na_values = "=")
In [17]:
df.Production.isnull().sum()
Out[17]:
the following heatmap helps to identify the null values in the dataframa¶
- all the white horizontal bars, which are white in color are null values
- color bar cbar is not requred though
In [19]:
sns.heatmap(df.isnull(), cmap = 'Blues_r')
Out[19]:
In [ ]:
df.dtypes
Out[ ]:
In [ ]:
df.shape
Out[ ]:
In [20]:
df.Production.isnull().count()
Out[20]:
In [ ]:
df.Production.isnull().sum()
In [ ]:
print ("Null values percentage : " , (3727/246091)*100)
dropna()¶
- deletes all rows, even if the row contains atleast a single null value (ie., across any column)
- Threshold is another parameter that can check the number of null values
- for example if there be a need to delete rows that has more than 4 null values
In [21]:
df.dropna(inplace = True)
In [22]:
df.shape
Out[22]:
Question: Which state has done well and which are not?¶
watch - Hans Roslng video from Ted on Data Visualisation
In [23]:
sns.kdeplot(df.Production)
Out[23]:
In [24]:
sns.boxplot(df.Production)
Out[24]:
In [25]:
sns.boxplot(df.Production, hue=df.State_Name)
Out[25]:
In [ ]:
sns.boxplot(df.Area)
Out[ ]:
In [ ]:
sns.kdeplot(df.Area)
Out[ ]:
In [ ]:
df[df.State_Name == 'Karnataka']
Out[ ]:
In [ ]:
df[df.State_Name == "Karnataka"]['District_Name'].unique()
Out[ ]:
In [ ]:
df_total = df.loc[df['District_Name'].str.contains('Total', case = False)]
In [ ]:
df_total.count()
Out[ ]:
to create a subset with crop type to be only Rice Crop¶
In [ ]:
df_total = df.loc[df['State_Name'].str.contains('Total', case = False)]
In [ ]:
df_total
Out[ ]:
In [ ]:
df_rice = df.loc[df['Crop'].str.contains('rice', case = False)]
In [ ]:
df_rice
Out[ ]:
to create a subset with only Rice and Wheat as crops¶
In [ ]:
df_rice_wheat = df.loc[df['Crop'].str.contains("wheat", case = False) |
df['Crop'].str.contains('rice', case = False)]
In [ ]:
df_rice_wheat
Out[ ]:
In [26]:
df.groupby(['State_Name', 'Crop', 'Crop_Year']).sum()
Out[26]:
In [27]:
df[df.State_Name == "West Bengal"]['Crop'].unique()
Out[27]:
In [ ]:
df.groupby(['State_Name', 'Crop_Year']).sum()
Out[ ]:
In [28]:
df_ = df.groupby(['State_Name', 'Crop_Year']).sum()
In [29]:
df_.head()
Out[29]:
In [30]:
df_.reset_index(inplace = True)
to get the Crop_Year Count in the dataset¶
In [31]:
df_[['State_Name', 'Crop_Year']].groupby('State_Name').count()
Out[31]:
In [ ]:
df[['State_Name', 'Crop']].groupby('State_Name').count()
Out[ ]:
In [ ]:
df_ani = df.loc[df['State_Name'].str.contains('Andaman and Nicobar Islands', case = False)]
In [ ]:
df_ani
Out[ ]:
In [ ]:
sns.lineplot(x='Crop_Year', y='Area', data = df[df.State_Name == 'Andaman and Nicobar Islands'])
Out[ ]:
In [ ]:
sns.lineplot(x='Crop_Year', y='Area', data = df[df.State_Name == 'Odisha'])
Out[ ]:
In [ ]:
sns.lineplot(x='Crop_Year', y='Area', data = df[df.State_Name == 'Andhra Pradesh'])
Out[ ]:
In [ ]:
sns.lineplot(x='Crop_Year', y='Production', data = df[df.State_Name == 'Andhra Pradesh'])
Out[ ]:
In [ ]:
sns.lineplot(x = 'Crop_Year', y = 'Production', data = df_, hue = 'State_Name')
Out[ ]:
place legend beside the graph so that they may not overlap¶
In [ ]:
import matplotlib.pyplot as plt
sns.lineplot(x = 'Crop_Year', y = 'Production', data = df_, hue = 'State_Name')
plt.legend(bbox_to_anchor = [1,1])
Out[ ]:
In [ ]:
# from matplotlib.cbook import boxplot_stats
sns.lineplot(x = 'Crop_Year', y = 'Production', data = df_, hue = 'State_Name')
plt.legend(bbox_to_anchor = (1, 1))
Out[ ]:
source for the following code:¶
https://stackoverflow.com/questions/47230817/plotly-notebook-mode-with-google-colaboratory
- simply pass "colab" as the value for the parameter renderer in fig.show(renderer="colab")
In [ ]:
import plotly.graph_objects as go
fig = go.Figure(data=[go.Bar(y=[2, 1, 3, 5, 8, 9, 12, 11, 2, 4, 2, 3])],
layout_title_text="A Figure Displayed with the 'colab' Renderer")
fig.show(renderer="colab")
In [32]:
!pip3 install plotly_express
In [33]:
import plotly_express as px
In [34]:
df.head(1)
Out[34]:
In [35]:
df_ = df.groupby(['State_Name', 'Crop_Year']).sum()
In [36]:
df_.head()
Out[36]:
In [37]:
df_.reset_index(inplace = True)
In [38]:
df_.head()
Out[38]:
In [ ]:
df_[['State_Name', 'Crop_Year']].groupby('State_Name').count()
Out[ ]:
plotly scatter Graph¶
In [47]:
df_.head(2)
Out[47]:
In [55]:
df_.sort_values('Crop_Year', inplace = True)
plotly graph by Area¶
In [56]:
# px.scatter(df_, x = "Area", y='Production', animation_frame = 'Crop_Year',
# animation_group = "State_Name", color = "State_Name")
import plotly.graph_objects as go
fig = go.Figure(px.scatter(df_, x = "Area", y = 'Production',
animation_frame = 'Crop_Year',animation_group = "State_Name",
color = "State_Name"))
# layout_title_text="Area on X and Production on Y, Animation frame as Crop Year grouped by State Name, Colors indicates states")
fig.show(renderer="colab")
In [57]:
import plotly.graph_objects as go
fig = go.Figure(px.bar(df_, x = "Area", y='Production',
animation_frame = 'Crop_Year',animation_group = "State_Name",
color = "State_Name"),
layout_title_text="Area on X and Production on Y, Animation frame as Crop Year grouped by State Name, Colors indicates states"
)
fig.show(renderer="colab")
In [58]:
import plotly.graph_objects as go
fig = go.Figure(px.bar(df_, x = "Crop_Year", y='Area', color = "Crop_Year"),
layout_title_text="Crop year on X and Area on Y, Crop Year as Colors indicates states"
)
fig.show(renderer="colab")
In [59]:
import plotly.graph_objects as go
fig = go.Figure(px.funnel(df_, x = "State_Name", y='Area', color = "State_Name"),
layout_title_text="State on X and Area on Y, Crop Year as Colors indicates states"
)
fig.show(renderer="colab")
In [60]:
import plotly.graph_objects as go
fig = go.Figure(px.funnel(df_, x = "State_Name", y='Area', color = "Crop_Year"),
layout_title_text="State on X and Area on Y, Crop Year as Colors indicates states"
)
fig.show(renderer="colab")
In [61]:
import plotly.graph_objects as go
fig = go.Figure(px.funnel(df_, x = "State_Name", y='Area'),
layout_title_text="State on X and Area on Y"
)
fig.show(renderer="colab")
Exercise from Plotly Webinar - Training session¶
In [62]:
df = px.data.gapminder()
In [63]:
df
Out[63]:
In [64]:
df = px.data.gapminder().query("year == 2007")
In [65]:
df
Out[65]:
In [66]:
df.head()
Out[66]:
In [67]:
configure_plotly_browser_state
import plotly.graph_objects as go
fig = go.Figure(px.strip(df, x = "lifeExp", hover_name = 'country', color = 'continent'))
fig.show(renderer="colab")
In [68]:
df = pd.read_csv('/content/agriculture_ds.csv', na_values = "=")
In [69]:
df.head(1)
Out[69]:
In [70]:
df.dropna(inplace = True)
In [71]:
df_[(df_.State_Name == 'Kerala') & (df_.Crop_Year == 2000)]
Out[71]:
In [72]:
df[(df.State_Name == 'Kerala') & (df.Crop_Year == 2000)].sort_values('Production')
Out[72]:
- as the weight of the coconut will be very heavier than when compared to others, this turns out to be outlie.
- so total production alone should not be considered but should be sliced into crops of similar nature
- eg: rice, wheat, maize, ragi, etc...
In [73]:
df_ = df[df.Crop.isin(['Rice', 'Wheat', 'Maize', "Ragi"])]
In [74]:
df_
Out[74]:
In [75]:
df_ = df[df.Crop.isin(['Rice', 'Wheat', 'Maize', "Ragi"])].groupby(['State_Name', 'Crop_Year']).sum()
In [76]:
df_
Out[76]:
In [77]:
df_.reset_index(inplace = True)
In [78]:
df_.head(1)
Out[78]:
In [79]:
df_.sort_values('Crop_Year', inplace = True)
In [80]:
import plotly.graph_objects as go
fig = go.Figure(px.scatter(df_, x='Area', y = 'Production',
animation_frame = 'Crop_Year',
animation_group = 'State_Name', color = 'State_Name'))
fig.show(renderer="colab")
In [81]:
df_['Efficiency'] = df_['Production'] / df_['Area']
In [82]:
df_.head(2)
Out[82]:
In [83]:
import plotly.graph_objects as go
fig = go.Figure(px.scatter(df_, x = 'Area', y = 'Efficiency', size = 'Production',
animation_frame = "Crop_Year", animation_group = "State_Name",
color = 'State_Name'))
fig.show(renderer="colab")
In [ ]:
df_.head(2)
fixing the points that are going out side the plot with range parameter¶
In [84]:
import plotly.graph_objects as go
fig = go.Figure(px.scatter(df_, x = 'Area', y = 'Efficiency', size = 'Production',
animation_frame = "Crop_Year", animation_group = "State_Name", range_y = [0.75, 5], range_x = [-1E6, 20E6],
color = 'State_Name'))
fig.show(renderer="colab")
In [85]:
df[(df.State_Name == 'Kerala') & (df.Crop_Year == 2000)].sort_values('Production')
Out[85]:
In [86]:
df[df.Crop.isin(['Rice', "Wheat", "Ragi", "Maize"])]
Out[86]:
In [96]:
df_ = df[df.Crop.isin(['Rice', "Wheat", "Ragi", "Maize"])].groupby(['State_Name', 'Crop_Year']).sum()
In [97]:
df_.head(2)
Out[97]:
In [99]:
df_.reset_index(inplace=True)
In [100]:
df_.sort_values('Crop_Year', inplace=True)
In [101]:
df_.head(2)
Out[101]:
In [105]:
fig = go.Figure(px.scatter(df_, x="Area", y='Production',
animation_frame = "Crop_Year",
animation_group='State_Name', color = "State_Name"))
fig.show(renderer="colab")
New Col by name Efficiency is created, and added to the parameter Size¶
In [107]:
df_['Efficiency'] = df_['Production'] / df_['Area']
In [109]:
fig = go.Figure(px.scatter(df_, x="Area", y='Efficiency', size='Production',
animation_frame = "Crop_Year",
animation_group='State_Name', color = "State_Name"))
fig.show(renderer="colab")
address the values that are going ## beyong the graph area
In [110]:
fig = go.Figure(px.scatter(df_, x="Area", y='Efficiency', size='Production',
animation_frame = "Crop_Year",
animation_group='State_Name', color = "State_Name",
range_y = [0.75, 5], range_x = [-1E6, 20E6]))
fig.show(renderer="colab")
some other exercises like:
- single state analysis
- two states analysis for comparative view - on a single crop or couple of more crops
- by the price values - as the cost of rice and cost of oil seeds is not the same
- only years for comparision
- with too much time, experiment with a comparative view of crop types and
- seaasons and their types.
In [ ]: