Consulting - Quality
Difference makes the DIFFERENCE
Agriculture DataSet
Bars, Boxes and Functions
In [1]:
!pip3 install plotly_express
In [2]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
import plotly_express as px
In [4]:
df = pd.read_csv('/content/clean_agri_ds.csv')
In [5]:
df.State_Name.unique()
Out[5]:
In [6]:
df.dtypes
Out[6]:
In [7]:
df.head(3)
Out[7]:
In [8]:
df[df.isna().any(axis=1)]
Out[8]:
In [9]:
df[df['Production'] == "="]
Out[9]:
In [10]:
df.drop(['Len State', 'Len Dist', 'Len Season', 'Len Crop'], inplace = True, axis=1)
In [11]:
df.head(3)
Out[11]:
In [12]:
df.describe()
Out[12]:
In [13]:
df.set_index(['Unnamed: 0'])
Out[13]:
In [14]:
df.rename(columns = {'Unnamed: 0':"ColIndex"}, inplace = True)
In [15]:
df.head(2)
Out[15]:
In [16]:
df.set_index(['ColIndex'])
Out[16]:
In [17]:
df.head(2)
Out[17]:
In [18]:
df.sort_values(['Production'], inplace = True, ascending = False)
In [19]:
df.head(2)
Out[19]:
Create subset for AP from India dataset¶
In [20]:
dfap = df[df['State_Name'] == "Andhra Pradesh"]
In [21]:
dfap.head(3)
Out[21]:
In [22]:
dfap_group = df[df['State_Name'] == "Andhra Pradesh"].groupby('District_Name').sum()
In [23]:
dfap_group
Out[23]:
In [24]:
dfap.head(2)
Out[24]:
In [25]:
dfap.rename(columns = {"Unnamed: 0":"ColIndex"}, inplace=True)
In [26]:
dfap.set_index('ColIndex')
Out[26]:
In [27]:
dfap.describe()
Out[27]:
In [28]:
dfap['Crop_Year'] = df['Crop_Year'].astype(str)
In [29]:
dfap['ColIndex'] = dfap['ColIndex'].astype(str)
In [30]:
dfap.describe()
Out[30]:
In [31]:
sns.lineplot(x="District_Name", y="Area", hue='Crop_Year', data = dfap)
plt.xticks(rotation = 90)
plt.legend(bbox_to_anchor = [1,1])
Out[31]:
In [32]:
sns.lineplot(x="District_Name", y="Production", hue='Crop_Year', data = dfap)
plt.xticks(rotation = 90)
plt.legend(bbox_to_anchor = [1,1])
Out[32]:
In [33]:
sns.barplot(x="District_Name", y="Area", data = dfap)
plt.xticks(rotation = 90)
Out[33]:
In [34]:
sns.barplot(x="District_Name", y="Production", data = dfap)
plt.xticks(rotation = 90)
Out[34]:
In [35]:
dfap['District_Name'].unique()
Out[35]:
In [36]:
dfap.describe()
Out[36]:
In [37]:
dfap.dtypes
Out[37]:
Working with Plot features in Pandas¶
- The .plot.* methods are applicable on both Series and DataFrames
- By default, each of the columns is plotted as a different element (line, boxplot,…)
- Any plot created by pandas is a Matplotlib object.
source: https://pandas.pydata.org/docs/getting_started/intro_tutorials/04_plotting.html
In [38]:
dfap.plot(figsize = (8,4))
Out[38]:
In [39]:
dfap.groupby('District_Name')['Area'].sum().plot.bar()
Out[39]:
In [40]:
dfap.groupby('District_Name').sum().plot(kind='line')
plt.xticks(rotation=90)
Out[40]:
In [41]:
dfap.sort_values(by="District_Name", inplace=True)
In [42]:
dfap.plot.scatter(x='Area', y='Production')
Out[42]:
In [43]:
dfap.plot.box()
Out[43]:
In [44]:
fig, axs = plt.subplots(figsize = (12, 6));
# dfap.plot.area(figsize = (12, 6), subplots = True)
dfap.plot.line(ax=axs, subplots = True, marker=".");
In [45]:
fig.savefig('Area_Production.png')
In [46]:
dfap
Out[46]:
In [47]:
dfap.sort_values("Production", ascending = True, inplace = True)
In [48]:
dfaps = dfap['Production'].cumsum()
In [49]:
dfaps.plot()
plt.xticks(rotation = 90)
Out[49]:
In [50]:
dfapa = dfap['Area'].cumsum()
In [51]:
dfapa.plot()
plt.xticks(rotation = 90)
Out[51]:
In [52]:
plt.figure()
dfap.Production.plot(kind='density')
Out[52]:
In [53]:
df2 = pd.DataFrame(np.random.rand(10, 4), columns = ['a', 'b', 'c', 'd'])
df2.plot.bar();
In [54]:
df2 = pd.DataFrame(np.random.rand(10, 4), columns = ['a', 'b', 'c', 'd'])
df2.plot.bar(stacked = True);
In [55]:
df2 = pd.DataFrame(np.random.rand(10, 4), columns = ['a', 'b', 'c', 'd'])
df2.plot.bar(stacked = True);
plt.legend(bbox_to_anchor = [1,1]) # to place legend outside the graph, towards the right top
Out[55]:
In [56]:
df2 = pd.DataFrame(np.random.rand(10, 4), columns = ['a', 'b', 'c', 'd'])
df2.plot.barh(stacked = True);
In [57]:
dfap['Production'].diff().hist()
Out[57]:
In [58]:
plt.figure()
dfap['Area'].diff().hist(color = 'k', bins=50)
Out[58]:
In [59]:
plt.figure()
dfap['Area'].diff().hist(color = 'k', bins=50)
Out[59]:
In [60]:
dfapb = dfap.sample(1000).groupby('Season').sum()
dfapb.plot.box()
Out[60]:
In [61]:
dfapb
Out[61]:
In [62]:
dfapb = dfap.sample(10)
dfapb.boxplot()
Out[62]:
In [63]:
dfap.sample(10).boxplot(by='Crop_Year', figsize=(10, 3))
plt.xticks(rotation=90)
Out[63]:
In [64]:
dfap.sample(10).boxplot(by='Season', figsize=(10, 3))
plt.xticks(rotation=90)
Out[64]:
In [65]:
dfap.sample(10).boxplot(by='Crop', figsize=(10, 3))
plt.xticks(rotation=90)
Out[65]:
In [66]:
dfap.head(2)
Out[66]:
In [67]:
dfap.sample(10).boxplot(by="Season");
plt.xticks(rotation=90);
In [68]:
dfap.sample(10).boxplot(by=["Season", 'Crop'],
figsize = (8,6), layout=(2, 1));
plt.xticks(rotation=90);
In [69]:
dfap.plot.scatter(x="Production", y='Area')
Out[69]:
In [70]:
dfap.plot.scatter(x="Production", y='Area', s=5)
Out[70]:
In [71]:
dfap.plot.scatter(x="Area", y='Production', s=2)
Out[71]:
In [72]:
dfap.sample(20).plot.hexbin('Area', 'Production', gridsize=15)
Out[72]:
In [73]:
dfap.sample(30).Production.plot();
dfap.sample(30).Area.plot(secondary_y = True, style = "o")
plt.tight_layout()
In [74]:
dfap.sample(100).plot(subplots = True, figsize = (6, 6))
Out[74]:
How to calculate summary statistics?¶¶
In [75]:
dfap.Production.mean()
Out[75]:
In [76]:
dfap.Area.mean()
Out[76]:
In [77]:
dfap[['Production', 'Area']].mean()
Out[77]:
In [78]:
dfap[['Production', 'Area']].describe()
Out[78]:
In [79]:
dfap.agg(
{
"Production": ["min", "max", "median", "skew"],
"Area": ["min", "max", "median", "mean"],
}
)
Out[79]:
In [80]:
dfap.head(2)
Out[80]:
- Aggregation statistics can be calculated on entire columns or rows
- groupby provides the power of the split-apply-combine pattern
- value_counts is a convenient shortcut to count the number of entries in each category of a variable.
In [81]:
dfap[['Season', 'Area']].groupby('Season').mean()
Out[81]:
In [82]:
dfap[['Crop', "Area"]].groupby('Crop').mean()
Out[82]:
In [83]:
dfap.groupby('District_Name').mean()
Out[83]:
In [84]:
dfap.groupby('District_Name').mean().sort_values('Area', ascending = False)
Out[84]:
In [85]:
dfap.groupby('District_Name').mean().sort_values('Production', ascending = False)
Out[85]:
In [86]:
dfaps = dfap.groupby('District_Name').mean().sort_values('Production', ascending = False)
In [87]:
dfaps.plot(figsize=(10, 5))
plt.xticks(rotation=90)
Out[87]:
In [88]:
dfap_season = dfap.groupby('Season')['Area'].mean()
In [89]:
dfap_season
Out[89]:
In [90]:
dfap_season.plot.box()
Out[90]:
In [91]:
dfap_season.plot.bar()
Out[91]:
In [92]:
dfTest = dfap.groupby('Season')['Area'].median()
ax = dfTest.plot.bar(figsize = (10, 4))
for p in ax.patches:
ax.annotate(np.round(p.get_height(), decimals = 2), (p.get_x() * 1.05, p.get_height() * 1.05))
In [93]:
dfap.groupby('Season')['Area'].mean().plot.bar()
Out[93]:
In [94]:
dfap.groupby("Crop")['Area'].mean().plot.bar(figsize = (10, 3))
Out[94]:
In [95]:
dfap.groupby("Crop")['Area'].median().plot.bar(figsize = (10, 3))
Out[95]:
In [96]:
dfTest = dfap.groupby('District_Name')['Area'].mean()
ax = dfTest.plot.bar(figsize = (10, 4))
for p in ax.patches:
ax.annotate(np.round(p.get_height(), decimals = 2), (p.get_x() * 1.05, p.get_height() * 1.05))
In [97]:
dfap.groupby('District_Name')['Area'].median().plot.bar(figsize = (10, 3))
Out[97]:
In [98]:
dfap.groupby(["District_Name", 'Season'])['Area'].mean().plot.bar(figsize = (10, 3))
Out[98]:
In [99]:
dfap.groupby(["Season", 'District_Name'])['Area'].mean().plot.bar(figsize = (10, 3))
Out[99]:
Placing values over bars in bar plot¶
In [100]:
dfTest = dfap.groupby(["Season", 'District_Name'])['Area'].mean()
ax = dfTest.plot.bar(figsize = (10, 4))
for p in ax.patches:
ax.annotate(str(p.get_height()), (p.get_x() * 1.05, p.get_height() * 1.05))
In [101]:
dfTest = dfap.groupby(["Season", 'District_Name'])['Area'].mean()
ax = dfTest.plot.bar(figsize = (12, 4))
for p in ax.patches:
ax.annotate(np.round(p.get_height(), decimals = 2), (p.get_x() * 1.05, p.get_height() * 1.05))
In [102]:
dfTest = dfap.groupby(["Season", 'District_Name'])['Area'].mean()
ax = dfTest.plot.barh(figsize = (4, 12))
#for p in ax.patches:
# ax.annotate(np.round(p.get_width(), decimals = 2), (p.get_y() * 1.05, p.get_width() * 1.05))
for i in ax.patches:
plt.text(i.get_width(), i.get_y()+.05,
str(round((i.get_width()), 2)),
fontsize = 10, fontweight ='bold',
color ='grey')
In [103]:
dfap.head()
Out[103]:
Bar plot with MatPlotLib¶
In [104]:
df.head(2)
Out[104]:
In [105]:
dfap.head(2)
Out[105]:
In [106]:
dfap.dtypes
Out[106]:
In [107]:
dfap.set_index("ColIndex", inplace = True)
In [108]:
dfap.head(2)
Out[108]:
In [109]:
dfapTest = dfap.groupby('District_Name').sum()
dfapTest.reset_index(inplace = True)
dfapTest
Out[109]:
In [110]:
fig = plt.figure(figsize = (10, 4))
dfapTest = dfap.groupby('District_Name').sum()
dfapTest.reset_index(inplace=True)
plt.bar(dfapTest['District_Name'], dfapTest['Production'])
plt.xticks(rotation = 90)
Out[110]:
In [111]:
fig = plt.figure(figsize = (10, 4))
dfapTest = dfap.groupby('District_Name').sum()
dfapTest.reset_index(inplace=True)
plt.barh(dfapTest['District_Name'], dfapTest['Production'])
plt.xticks(rotation = 90)
Out[111]:
In [112]:
dfapTest = dfap.groupby('District_Name').sum()
dfapTest.reset_index(inplace=True)
plt.figure(figsize = (15, 3))
plt.subplot(131)
plt.bar(dfapTest['District_Name'], dfapTest['Production'])
plt.xticks(rotation = 90)
plt.subplot(132)
plt.scatter(dfapTest['District_Name'], dfapTest['Production'])
plt.xticks(rotation = 90)
plt.subplot(133)
plt.plot(dfapTest['District_Name'], dfapTest['Production'], linewidth=4)
plt.xticks(rotation = 90)
Out[112]:
In [113]:
dfapTest = dfap.groupby('District_Name').sum()
dfapTest.reset_index(inplace=True)
plt.figure(figsize = (15, 3))
plt.subplot(131)
plt.bar(dfapTest['District_Name'], dfapTest['Area'])
plt.xticks(rotation = 90)
plt.subplot(132)
plt.scatter(dfapTest['District_Name'], dfapTest['Area'])
plt.xticks(rotation = 90)
plt.subplot(133)
plt.plot(dfapTest['District_Name'], dfapTest['Area'], linewidth=4)
plt.xticks(rotation = 90)
Out[113]:
In [114]:
dfapTest = dfap.groupby('Crop').sum()
dfapTest.reset_index(inplace = True)
In [115]:
dfapTest.sort_values('Production', inplace = True, ascending=False)
In [116]:
plt.figure(figsize = (12, 4))
plt.bar(dfapTest['Crop'], dfapTest["Production"])
plt.xticks(rotation=90);
dataframe dfapNC to exclude Coconut Crop - as it is dominant crop¶
In [117]:
# dfapTestNoCoconut = dfapNC
dfapNC = dfap.loc[~dfap['Crop'].str.contains('Coconut', case = False)]
In [118]:
dfapNC
Out[118]:
In [119]:
dfapNC.groupby('Crop').sum()
dfapNC.reset_index(inplace = True)
plot excluding Coconut as it is the most dominating crop¶
In [120]:
plt.figure(figsize = (10, 4));
plt.bar(dfapNC["Crop"], dfapNC['Production']);
plt.xticks(rotation = 90);
excluding Sugarcane as well as it the next biggest crop after Coconut¶
In [121]:
dfapNcSc = dfapNC[~ dfapNC['Crop'].str.contains('Sugarcane', case = False)]
In [122]:
dfapNcSc = dfapNcSc[~ dfapNcSc['Crop'].str.contains('Rice', case = False)]
In [123]:
dfapNcSc
Out[123]:
In [124]:
dfapNcSc.reset_index(inplace = True)
In [125]:
dfapncsc = dfapNcSc.groupby('Crop').sum()
In [126]:
dfapncsc.reset_index(inplace = True)
In [127]:
plt.figure(figsize = (12, 4));
plt.bar(dfapncsc['Crop'], dfapncsc['Production']);
plt.xticks(rotation = 90);
# plt.xlabel("Crop Names")
#plt.ylabel("Production")
# plt.title(label="excludes Coconut, Sugarcane, Rice", fontsize=20, color='green')
In [128]:
plt.figure(figsize = (12, 4));
plt.bar(dfapncsc['Crop'], dfapncsc['Area']);
plt.xticks(rotation = 90);
In [129]:
dfapncsc
Out[129]:
In [130]:
dfap_season
Out[130]:
In [131]:
dfapseason = dfap.groupby('Season').sum()
In [132]:
dfapseason
Out[132]:
In [133]:
dfapseason.reset_index(inplace = True)
In [134]:
plt.bar(dfapseason['Season'], dfapseason['Production'])
Out[134]:
In [135]:
plt.bar(dfapseason['Season'], dfapseason['Area'])
Out[135]:
In [136]:
dfapds = dfap.groupby(['District_Name', 'Season']).sum()
In [137]:
dfapds
Out[137]:
In [138]:
dfapds.reset_index(inplace = True)
In [139]:
dfapds.to_excel('dfapds_for_graph.xlsx')
In [140]:
df.head()
Out[140]:
In [141]:
df.drop('ColIndex', inplace = True, axis=1)
In [142]:
df.head()
Out[142]:
In [143]:
dfap.head(2)
Out[143]:
In [144]:
dfap.reset_index(inplace = True)
In [145]:
dfap.dtypes
Out[145]:
In [146]:
dfap.drop('ColIndex', inplace = True, axis=1)
In [147]:
dfap.head(2)
Out[147]:
In [148]:
dfapgrp = dfap.groupby(['Crop_Year', 'District_Name', 'Season', 'Crop']).sum()
In [149]:
dfapgrp.head(2)
Out[149]:
In [150]:
dfapgrp.reset_index(inplace = True)
In [151]:
dfapgrp.shape
Out[151]:
In [152]:
dfapgrp.head(2)
Out[152]:
In [153]:
dfapgrp.to_excel('dfapgrp.xlsx')
In [155]:
dfapgrp.head(2)
Out[155]:
In [157]:
plt.bar(dfapgrp['Crop_Year'], dfapgrp['Area'])
plt.xticks(rotation = 90)
Out[157]:
In [168]:
dfaptest = dfapgrp.groupby(['Crop_Year', 'Season'])['Area'].sum()
ax = dfaptest.plot.bar(figsize = (15, 4), fontsize=10)
for p in ax.patches:
ax.annotate(np.round(p.get_height(), decimals = 2), (p.get_x() * 1.05, p.get_height() * 1.05))
In [177]:
dfaptest = dfapgrp.groupby(['Crop_Year', 'Season'])['Area'].sum()
ax = dfaptest.plot.barh(figsize = (15, 20), fontsize=10)
for i in ax.patches:
plt.text(i.get_width(), i.get_y()+.05,
str(round((i.get_width()), 2)),
fontsize = 10, fontweight ='bold',
color ='grey')
In [190]:
dftest = dfapgrp.loc[(dfapgrp['District_Name'] == 'GUNTUR')];
dftest.groupby(['Crop_Year', 'Season'])['Area'].sum().plot.bar(figsize = (10, 4));
plt.xticks(rotation = 90);
Function to plot graph¶
In [191]:
def plotDistGraph(distName):
dftest = dfapgrp.loc[(dfapgrp['District_Name'] == distName)];
dftest.groupby(['Crop_Year', 'Season'])['Area'].sum().plot.bar(figsize = (10, 4));
plt.xticks(rotation = 90);
In [192]:
dfapgrp.District_Name.unique()
Out[192]:
In [194]:
plotDistGraph('VIZIANAGARAM')
In [195]:
plotDistGraph('SRIKAKULAM')
In [200]:
print(dfapgrp.min())
In [201]:
print(dfapgrp.max())
In [220]:
dfTest = dfapgrp[(dfapgrp['District_Name'] == "ANANTAPUR") &
(dfapgrp['Season'] == "Rabi")]
In [221]:
dfTest
Out[221]:
In [231]:
def plotDistGraph(distName, season):
dfTest = dfapgrp[(dfapgrp['District_Name'] == distName) &
(dfapgrp['Season'] == season)]
dfTest.groupby(['Crop_Year', 'Season'])['Production'].sum().plot.bar(figsize = (10, 4));
plt.xticks(rotation = 90);
In [232]:
plotDistGraph('ANANTAPUR', 'Rabi')
In [224]:
plotDistGraph('ANANTAPUR', 'Kharif')
In [225]:
dfapgrp.District_Name.unique()
Out[225]:
In [227]:
plotDistGraph('VIZIANAGARAM', 'Rabi')
In [228]:
plotDistGraph('VIZIANAGARAM', 'Whole Year')
In [233]:
def plotDistGraph(distName, season):
dfTest = dfapgrp[(dfapgrp['District_Name'] == distName) &
(dfapgrp['Season'] == season)]
dfTest.groupby(['Crop'])['Production'].sum().plot.bar(figsize = (10, 4));
plt.xticks(rotation = 90);
In [236]:
dfapgrp.District_Name.unique()
Out[236]:
In [238]:
plotDistGraph('PRAKASAM', 'Whole Year')
In [239]:
plotDistGraph('GUNTUR', 'Whole Year')
In [240]:
plotDistGraph('KURNOOL', 'Whole Year')
In [245]:
def plotGraph(stateName, distName, season):
dfTest = df[(df['State_Name'] == stateName) &
(df['District_Name'] == distName) &
(df['Season'] == season)]
dfTest.groupby(['Crop'])['Production'].sum().plot.bar(figsize = (10, 4));
plt.xticks(rotation = 90);
In [243]:
df.State_Name.unique()
Out[243]:
In [247]:
plotGraph('Andhra Pradesh', 'GUNTUR', 'Rabi')
In [248]:
plotGraph('Andhra Pradesh', 'GUNTUR', 'Kharif')
In [249]:
plotGraph('Andhra Pradesh', 'GUNTUR', 'Whole Year')
In [255]:
dfTest = df[(df['State_Name'] == 'Andhra Pradesh') &
(df['District_Name'] == 'GUNTUR') &
(df['Season'] == 'Whole Year')].groupby('Crop').sum()
In [256]:
dfTest
Out[256]:
In [ ]: