Elements of Data Science

import pandas as pd

df = pd.DataFrame({"Name": ["John", "Bob", "Jim", "Kate"], "Job": ["Accountant", "Programmer", "Programmer", "Marketing"]})
df_1 = pd.DataFrame({"VP": ["Tom", "Andy", "Kate"], "Job": ["Accountant", "Programmer", "Marketing"]})

df_merged = df.merge(df_1, on="Job", how="inner")

print(df_merged)

import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt

df = pd.DataFrame({"Name": ["John", "Bob", "Jim", "Kate"], "Job": ["Accountant", "Programmer", "Programmer", "Marketing"], "Salary": [1000, 2500, 2750, 1800]})
df_1 = pd.DataFrame({"VP": ["Tom", "Andy", "Kate"], "Job": ["Accountant", "Programmer", "Marketing"]})

df_merged = df.merge(df_1, on="Job", how="inner")

print(df_merged["Job"].value_counts())
sb.distplot(df_merged["Job"].value_counts())
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer

dataset = load_breast_cancer()
cols = [
    'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19',
    'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'V29',
    'V30', 'V31', 'V32', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V39'
]

df = pd.DataFrame(dataset['data'], columns=cols)
df['target'] = dataset.target

# show first a few rows
print(df.head())
# show data type for each column
print(df.info())
# show summary statistics for each columns
print(df.describe())
# check the target variable properties
print(df['target'].value_counts())

# Density Plot
df['V11'].plot.kde()
plt.show()
# Histogram
df['V11'].plot.hist()
plt.show()
# Box Plot
df.boxplot(['V11'])
plt.show()
# Scatter Plots (detecting relationship between variables)
df.plot.scatter(x='V11', y='V12')
plt.show()
# Scatter Matrix Plot
pd.plotting.scatter_matrix(df[['V11', 'V21', 'V31']], figsize=(15,15))
plt.show()

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer

dataset = load_breast_cancer()
cols = [
    'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19',
    'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'V29',
    'V30', 'V31', 'V32', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V39'
]

df = pd.DataFrame(dataset['data'], columns=cols)

col = ['V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19']
heatmap = np.corrcoef(df[col].values.T)

fig, ax = plt.subplots(figsize=(15, 15))
im = ax.imshow(heatmap, cmap='PiYG', vmin=1)
fig.colorbar(im)
ax.grid(False)
[[ax.text(j, i, round(heatmap[i, j], 2), ha="center", va="center", color="w") for j in range(len(heatmap))] for i in
 range(len(heatmap))]

ax.set_xticks(np.arange(len(col)))
ax.set_yticks(np.arange(len(col)))
ax.set_xticklabels(col)
ax.set_yticklabels(col)

plt.show()

import pandas as pd
import numpy as np
import seaborn
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer

dataset = load_breast_cancer()
cols = [
    'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19',
    'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'V29',
    'V30', 'V31', 'V32', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V39'
]

df = pd.DataFrame(dataset['data'], columns=cols)

col = ['V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19']
heatmap = np.corrcoef(df[col].values.T)

seaborn.heatmap(heatmap, yticklabels=col, xticklabels=col, cmap='PiYG', annot=True)
plt.show()

import pandas as pd
from sklearn.preprocessing import LabelEncoder

df = pd.DataFrame([
    ['house', 3, 2572, 'S', 1372000, 'Y'],
    ['apartment', 2, 1386, 'N', 699000, 'N'],
    ['house', 3, 1932, 'L', 800000, 'N'],
    ['house', 1, 851, 'M', 451000, 'Y'],
    ['apartment', 1, 600, 'N', 324000, 'N']
])

df.columns = ['type', 'bedrooms', 'area', 'garden_size', 'price', 'loan_approved']
print(df)

# Converting garden_size using mapping
mapping = dict({'N': 0, 'S': 5, 'M': 10, 'L': 20})
df['num_garden_size'] = df['garden_size'].map(mapping)

# Converting label loan_approved using LabelEncoder
loan_enc = LabelEncoder()
df['num_loan_approved'] = loan_enc.fit_transform(df['loan_approved'])

print(df)

from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
import pandas as pd

df = pd.DataFrame({"Fruits": ['Apple', 'Banana', 'Banana', 'Mango', 'Banana']})

type_labelenc = LabelEncoder()
num_type = type_labelenc.fit_transform(df["Fruits"])

print(num_type) 
# output: 
# [0 1 1 2 1]

print(num_type.reshape(-1, 1)); 
# output: 
# [[0]
# [1]
# [1]
# [2]
# [1]]


type_enc = OneHotEncoder()
type_trans = type_enc.fit_transform(num_type.reshape(-1, 1)).toarray()

print(type_trans)
# output:
# [[1. 0. 0.]
# [0. 1. 0.]
# [0. 1. 0.]
# [0. 0. 1.]
# [0. 1. 0.]]

import pandas as pd

df = pd.DataFrame({"Fruits": ['Apple', 'Banana', 'Banana', 'Mango', 'Banana']})

dummies = pd.get_dummies(df)
print(dummies)

import pandas as pd

df = pd.DataFrame({
    "Fruits": ["Banana", "Apple", "Mango", "Mango", "Apple"],
    "Number": [5, None, 3, None, 1]
})

# Display the total number of missing values for each column
print(df.isnull().sum())

# Display the total number of missing values for each row
print(df.isnull().sum(axis=1))

import pandas as pd

df = pd.DataFrame({
    "Fruits": ["Banana", "Apple", "Mango", "Mango", "Apple"],
    "Number": [5, None, 3, None, 1]
})

# Drop the rows with null values
print(df.dropna())

# Drop the columns with null values
print(df.dropna(axis=1))

from sklearn.impute import SimpleImputer
import numpy as np

arr = np.array([
    [5, 3, 2, 2],
    [3, None, 1, 9],
    [5, 2, 7, None]
])

imputer = SimpleImputer(strategy='mean')
imp = imputer.fit_transform(arr)
print(imp)

from sklearn.preprocessing import StandardScaler
import numpy as np

arr = np.array([
    [5, 3, 2, 2],
    [2, 3, 1, 9],
    [5, 2, 7, 6]
], dtype=float)

scale = StandardScaler()
print(scale.fit_transform(arr))
print(scale.scale_)

from sklearn.preprocessing import MinMaxScaler
import numpy as np

arr = np.array([
    [5, 3, 2, 2],
    [2, 3, 1, 9],
    [5, 2, 7, 6]
], dtype=float)

scale = MinMaxScaler()
print(scale.fit_transform(arr))
print(scale.scale_)

from sklearn.preprocessing import PolynomialFeatures
import numpy as np
import pandas as pd

df = pd.DataFrame({'a': np.random.rand(5), 'b': np.random.rand(5)})

cube = PolynomialFeatures(degree=3)
cube_features = cube.fit_transform(df)

cube_df = pd.DataFrame(cube_features, columns=[
    '1', 'a', 'b', 'a^2', 'ab', 'b^2', 'a^3', 'ba^2', 'ab^2', 'b^3'
])

print(cube_df)