import os
import pandas as pd

import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
from ipywidgets import interact

os.chdir('D:/Documents/MSc course/MAT002/coursework1')
a = pd.read_csv('price_quote_2017_09.csv')
a.head()
headings = a.columns

def region_code(x):
    if x==2:
        return 'London'
    elif x==1:
        return 'Catalogue'
    elif x==3:
        return 'South-East'
    elif x==4:
        return 'South-West'
    elif x==5:
        return 'East Anglia'
    elif x==6:
        return 'East Midlands'
    elif x==7:
        return 'West Midlands'
    elif x==8:
        return 'Yorks & Humber'
    elif x==9:
        return 'North-West'
    elif x==10:
        return 'North'
    elif x==11:
        return 'Wales'
    elif x==12:
        return 'Scotland'
    elif x==13:
        return 'Northern Ireland'
    else:
        return x
    
def shop_code(x):
    if x==1:
        return 'Chain of stores'
    elif x==2:
        return 'Independent'
    else:
        return x

a['REGION'] = a['REGION'].apply(lambda x: region_code(x))
a['SHOP_TYPE'] = a['SHOP_TYPE'].apply(lambda x: shop_code(x))

a.head()

a.groupby(by='ITEM_DESC').count()['PRICE'].sort_values(ascending=True)

def confidence_intervals(x):
    result = a[a['ITEM_DESC']==x]
    result = result[result['PRICE']>0]
    
    d = 1.0*np.array(result['PRICE'])
    n = len(d)
    m, se = np.mean(d), stats.sem(d)
    h = se * 2.575
    return m, m-h, m+h

def statistical(x):
    plt.clf()
    result = a[a['ITEM_DESC']==x]
    result = result[result['PRICE']>0]
    
    sns.distplot(result['PRICE'],kde=False, fit=stats.norm)
    plt.tight_layout()
    plt.xlabel('Price of a {}, £'.format(str.capitalize(x)))
    plt.ylabel('Probability distribution')
    plt.axvline(confidence_intervals(x)[0])
    plt.axvline(confidence_intervals(x)[1],linestyle='--')
    plt.axvline(confidence_intervals(x)[2],linestyle='--')
    plt.savefig(x+'_normal.png')
    plt.show()
    
    print('Mean price = £{:.2f}'.format(np.mean(result['PRICE'])))
    print('Lower confidence interval = £{:.2f}'.format(confidence_intervals(x)[1]))
    print('Upper confidence interval = £{:.2f}'.format(confidence_intervals(x)[2]))
    print('Number of quotes = {}'.format(len(result)))
    # Shapiro-Wilk test for normality
    print('Shapiro-Wilk test = {}'.format(stats.shapiro(result['PRICE'])))

def summarise(x):
    result = a[a['ITEM_DESC']==x]
    result = result[result['PRICE']>0]
    return result['PRICE'].describe()

interact(statistical, x=['LAGER - PINT 3.4-4.2%','LIQUEUR PER NIP   SPECIFY ML'])

summarise("HOME KILLED BEEF-LEAN MINCE KG")

S_W_tests = []

for i in pd.unique(a['ITEM_DESC']):

    result = a[a['ITEM_DESC']==i]
    result = result[result['PRICE']>0]
    
    # Looking at all ANOVA values
    S_W_tests.append((i,stats.shapiro(result['PRICE'])[0],stats.shapiro(result['PRICE'])[1]))
    
Swilk_stats = pd.DataFrame(S_W_tests,columns=['Item','T_Cr','p_val'])
Swilk_stats[['Item','T_Cr','p_val']].sort_values(by='p_val',ascending=True)

def wilcoxon(x):
    plt.clf()
    result = a[a['ITEM_DESC']==x]
    result = result[result['PRICE']>0]
    N = len(result)
    
    sns.distplot(result['PRICE'],kde=False, fit=stats.norm)
    plt.xlabel('Price of a {}, £'.format(str.capitalize(x)))
    plt.ylabel('Probability distribution')
    plt.axvline(confidence_intervals(x)[0])
    plt.axvline(confidence_intervals(x)[1],linestyle='--')
    plt.axvline(confidence_intervals(x)[2],linestyle='--')
    plt.show()
    
    print('Mean price = £{:.2f}'.format(np.mean(result['PRICE'])))
    print('Median price = £{:.2f}'.format(np.median(result['PRICE'])))
    print('Lower confidence interval (normal approx.) = £{:.2f}'.format(confidence_intervals(x)[1]))
    print('Upper confidence interval (normal approx.) = £{:.2f}'.format(confidence_intervals(x)[2]))
    # Shapiro-Wilk test for normality
    print('Shapiro-Wilk test = {}'.format(stats.shapiro(result['PRICE'])))
    print('Sample size = {}'.format(len(result['PRICE'])))
    
    #Wilcoxon signed-rank test
    test, p_val = stats.wilcoxon(x=np.median(result['PRICE'])*np.ones(shape=(N)), y=result['PRICE'])
    print('Wilcoxon Test stat = {}'.format(test))
    print('Wilcoxon P_value = {}'.format(p_val))
    
    confidence = []
    for i in np.linspace(np.median(result['PRICE'])*1, np.median(result['PRICE'])*1.5, num=66):
        p_val = stats.wilcoxon(x=i*np.ones(shape=(N)), y=result['PRICE'])[1]
        confidence.append((i,p_val))
    wilcoxon = pd.DataFrame(confidence,columns=['Medians','p_val'])
    return wilcoxon

wilcoxon("HOURLY RATE FOR SOLICITOR")

def regional_chart(x):
    plt.clf()
    result = a[a['ITEM_DESC']==x]
    result = result[result['PRICE']>0]
    result = result[result['REGION']!='Northern Ireland']
    
    g = result.groupby('REGION')
    
    sns.set(style="ticks")

    # Initialize the figure
    f, ax = plt.subplots(figsize=(7, 6))
    
    sns.boxplot(y="REGION", x="PRICE", data=result)
    sns.swarmplot(y="REGION", x="PRICE", data=result, size=2, color='.3')
    # Add in points to show each observation
    
    plt.title('Boxplot for {}'.format(str.capitalize(x)))
    plt.ylabel('Region code')
    plt.xlabel('Price, £')
    plt.savefig(x+'_regional.png')
    plt.show()
    
    print(g['PRICE'].mean())
    
    # ANOVA
    
    samples = [np.array(x[1]['PRICE']) for x in g]
    f_val, p_val = stats.kruskal(*samples)
    print('F value: {:.3f}, p value: {:.3f}'.format(f_val, p_val))

regional_chart('TAKEAWAY CHICKEN & CHIPS')

F_Stats = []

for i in pd.unique(a['ITEM_DESC']):

    result = a[a['ITEM_DESC']==i]
    result = result[result['PRICE']>0]
    
    # Looking at all ANOVA values
    g = result.groupby('REGION')
    samples = [np.array(x[1]['PRICE']) for x in g]
    f_val, p_val = stats.kruskal(*samples)
    
    F_Stats.append((i,f_val,p_val))

F_Stats = pd.DataFrame(F_Stats,columns=['Item','f_val','p_val'])

F_Stats.sort_values(by='p_val',ascending=False).head(n=30)

len(F_Stats[F_Stats['p_val']<0.05])

shoptype_chart('MENS SOCKS 1 PAIR')

F_Stats = []

for i in pd.unique(a['ITEM_DESC']):

    result = a[a['ITEM_DESC']==i]
    result = result[result['PRICE']>0]
    
    # Looking at all ANOVA values
    g = result.groupby('SHOP_TYPE')
    samples = [x[1]['PRICE'] for x in g]
    
    if len(samples)>1:
        f_val, p_val = stats.f_oneway(*samples)

        F_Stats.append((i,f_val,p_val))

F_Stats = pd.DataFrame(F_Stats,columns=['Item','f_val','p_val'])

F_Stats.sort_values(by='p_val',ascending=True)

b = pd.concat(pd.read_csv(i,names=headings,skiprows=1) for i in os.listdir() if i.startswith('upload'))
b.head()

import datetime as dt
b['QUOTE_DATE'] = b['QUOTE_DATE'].apply(lambda x: dt.datetime.strftime(dt.datetime.strptime(str(x)[-2:],'%m'),'%b'))
b.head()

def seasonal_analysis(x):
    
    plt.clf()
    
    result = b[b['ITEM_DESC']==x]
    result = result[result['PRICE']>0]
    
    # Initialize the figure
    f, ax = plt.subplots(figsize=(7, 6))
    
    sns.boxplot(y="PRICE", x="QUOTE_DATE", data=result)
    sns.swarmplot(y="PRICE", x="QUOTE_DATE", data=result, size=2, color='.3')
    # Add in points to show each observation
    
    plt.title('Boxplot for {}'.format(str.capitalize(x)))
    plt.ylabel('Price, £')
    plt.xlabel('Month of 2016')

    plt.show()

    g = result.groupby('QUOTE_DATE')
    
    # ANOVA
    
    samples = [np.array(x[1]['PRICE']) for x in g]
    
    # Look at min length of array

    minimum_sample = np.min([len(i) for i in samples])

    # Randomly remove

    samples = [np.random.choice(i,minimum_sample) for i in samples]
    
    #Friedman
    f_val, p_val = stats.friedmanchisquare(*samples)
    print('F value: {:.3f}, p value: {:.3f}'.format(f_val, p_val))
    
    print(g['PRICE'].mean())

seasonal_analysis("""KEROSENE - 1000L DELIVERED""")

F_Stats = []

for i in pd.unique(b['ITEM_DESC']):

    try:
        result = b[b['ITEM_DESC']==i]
        result = result[result['PRICE']>0]

        # Looking at all ANOVA values
        g = result.groupby('QUOTE_DATE')
        samples = [np.array(x[1]['PRICE']) for x in g]

        minimum_sample = np.min([len(i) for i in samples])

        samples = [np.random.choice(i,minimum_sample) for i in samples]

        f_val, p_val = stats.friedmanchisquare(*samples)

        F_Stats.append((i,f_val,p_val))
    
    except ValueError:
        pass
    
F_Stats = pd.DataFrame(F_Stats,columns=['Item','f_val','p_val'])

F_Stats.sort_values(by='p_val',ascending=True).head(n=20)