#!/usr/bin/env python
# coding: utf-8

# Obtain mass [kg] for production, imports and exports of PRODCOM codes
# 
# Only a fraction of production, imports and exports of PRODCOM are provided in
# kg. A bigger fraction is represented in physical units, but not all. This code
# conciliates PRODCOM and COMTRADE data to obtain production, imports and
# exports data in kg for as much PRODCOM entries as possible. This entices (when
# there is no PRODCOM data in kg):
#
# 1. Using "physical" conversion factors when PRODCOM and COMTRADE data are
# reported in the same units (and COMTRADE additionally reports data in kg).
#
# 2. Using conversion factors based on monetary data when PRODCOM and COMTRADE
# do not report data in the same units.
#
# 3. Convert monetary data to physical data when PRODCOM does not report data in
# physical units.
# 
# The output of this code (`Prodcom-ComtradeCombined.csv`) is used to generate
# "PRC_obs" by `Generate_observations_from_PRODCOM.py`.

## Import the data

import pandas as pd
import math
import numpy as np
import sys 

# Import the raw data from prodcom

pcData2018=pd.read_excel('data/raw_data/pc-2018-full.xlsx',
                         sheet_name='k15830722.xlsx 1', skiprows = range(1, 8))
pcData2018.columns=['Code','P-QTY','I-QTY','E-QTY','P-EUR','I-EUR','E-EUR']
pcLabels2018=pd.read_excel('data/pc-names.xlsx', sheet_name='labels', skiprows = range(1, 1))
pcUnits2018=pd.read_excel('data/pc-names.xlsx', sheet_name='units', skiprows = range(1, 1))
pcLabels2018.columns=['Code','Label']
pcLabels2018['Code']=pcLabels2018.Code.astype(str)
pcUnits2018.columns=['Code','PC unit']
pcUnits2018['Code']=pcUnits2018.Code.astype(str)

# Merge the Data with the names
pcData2018=pd.merge(pcData2018,pcLabels2018,how='left',on='Code')
pcData2018=pd.merge(pcData2018,pcUnits2018,how='left', on='Code')

# Import the conversion table from prodcom to combined numenclature/harmonised system
nomConvTable=pd.read_excel('data/raw_data/PRODCOM_2017-2018-CN_2018_mapping.xlsx',
                           sheet_name='CN_PRC2018',usecols=['PRCCODE','CNCODE'])

# Import the raw data from UN trade statistics (using the imports data for now)
ctUseCols=['Aggregate Level','Commodity Code','Qty Unit','Qty','Netweight (kg)','Trade Value (US$)']
comtradeImp2018=pd.read_csv('data/raw_data/ct-2018-imports.csv',usecols=ctUseCols)
comtradeExp2018=pd.read_csv('data/raw_data/ct-2018-exports.csv',usecols=ctUseCols)
comtradeImp2018.columns=['Agg Level','Code','Unit','I-QTY','I-kg','I-USD']
comtradeExp2018.columns=['Agg Level','Code','Unit','E-QTY','E-kg','E-USD']
# We only care about the highest aggregate level for this
comtradeImp2018=comtradeImp2018[comtradeImp2018['Agg Level']==6].reset_index(drop=True)
comtradeExp2018=comtradeExp2018[comtradeExp2018['Agg Level']==6].reset_index(drop=True)


### Combine COMTRADE into PRODCOM to make a big data frame

# List of units of measure that are "known" - correspondence between prodcom and comtrade names for these units
unitCorrsDict={'m'+chr(179): 'Volume in cubic meters', # Unicode code for ^3
               'm'+chr(178): 'Area in square metres', # Unicode code for ^2
               'p/st': 'Number of items',
               'pa': 'Number of pairs',
               'l': 'Volume in litres',
               '1 000 kWh': 'Electrical energy in thousands of kilowatt-hours',
               'm': 'Length in metres',
               'c/k': 'Weight in carats',
               'kg': 'Weight in kilograms',
              }

pcData2018['Com Code']=''
pcData2018['Com I-QTY']=0.0
pcData2018['Com E-QTY']=0.0
pcData2018['Com I-Unit']=''
pcData2018['Com E-Unit']=''
pcData2018['Com I-kg']=0.0
pcData2018['Com E-kg']=0.0
pcData2018['Com I-USD']=0.0
pcData2018['Com E-USD']=0.0
pcData2018['Error']=''


for i, currPcCode in enumerate(pcData2018['Code']):
        
    err=[]
    
    # Find the corresponding (CN) Comtrade codes for the Current Prodcom Code
    cNCodesList=nomConvTable[nomConvTable['PRCCODE']==str(currPcCode)]['CNCODE'].reset_index(drop=True)
    ctCode=[0]*len(cNCodesList)
    
    # Format the CN code to be compatible with the comtrade table (i. e. the comtrade table does not use the last two digits)
    try: 
        for j in enumerate(cNCodesList): ctCode[j[0]]=int(j[1][:-2].replace(' ', ''))
    except: 
       # print('Empty entry in conversion table, prodcom code: '+str(currPcCode))
        err.append('No entry in conversion table')
        continue
            
    # Go through the relevant CN codes in the Comtrade list and add up the values
    sumIQTY=0.0
    sumEQTY=0.0
    sumIkg=0.0
    sumEkg=0.0
    sumIUSD=0.0
    sumEUSD=0.0
    
    unit=[]
    ctRowDict=dict()
    ctRowDict2=dict()
    
    for j in enumerate(np.unique(ctCode)):
        # Find the corresponting row in the comtrade statistics
        try: ctRowDict=comtradeImp2018[comtradeImp2018['Code']==str(j[1])].to_dict('records')[0]
        except: 
            #print('Could not find Import comtrade code: '+str(j[1])+'; prodcom code: '+str(currPcCode))
            err.append('No data Comtrade - I '+str(j[1]))
            continue
        unit.append(ctRowDict['Unit']) 
        sumIQTY+=float(ctRowDict['I-QTY'])
        sumIkg+=float(ctRowDict['I-kg'])
        sumIUSD+=float(ctRowDict['I-USD'])
    
    if len(set(unit))>1:
        sumIQTY='NaN'
        #print('Found non-unique Comtrade Units for prodcom code: '+str(currPcCode))
        unit.append('Conflict')
        err.append('Non-unique Comtrade Units - Imp ')
        
    else:
        if len(set(unit))==0:
            unit.append('No unit found')
            
    Eunit=[]
    
    for j in enumerate(np.unique(ctCode)):    
        
        try: ctRowDict2=comtradeExp2018[comtradeExp2018['Code']==str(j[1])].to_dict('records')[0] 
        except: 
            #print('Could not find Export comtrade code: '+str(j[1])+'; prodcom code: '+str(currPcCode))
            err.append('No data Comtrade - E '+str(j[1]))
            continue
        Eunit.append(ctRowDict2['Unit'])
        sumEQTY+=float(ctRowDict2['E-QTY'])
        sumEkg+=float(ctRowDict2['E-kg'])
        sumEUSD+=float(ctRowDict2['E-USD'])
        
    if len(set(unit))>1:
        sumEQTY='NaN'
        Eunit.append('Conflict')
       # print('Found non-unique Comtrade Units for prodcom code: '+str(currPcCode))
        err.append('Non-unique Comtrade Units - Exp ')
    else:
        if len(set(Eunit))==0:
            Eunit.append('No unit found')

    pcData2018.at[i,'Com Code']=str(ctCode)
    pcData2018.at[i,'Com I-QTY']=sumIQTY
    pcData2018.at[i,'Com E-QTY']=sumEQTY
    pcData2018.at[i,'Com I-Unit']=str(unit[-1])
    pcData2018.at[i,'Com E-Unit']=str(Eunit[-1])
    pcData2018.at[i,'Com I-kg']=sumIkg
    pcData2018.at[i,'Com E-kg']=sumEkg
    pcData2018.at[i,'Com I-USD']=sumIUSD
    pcData2018.at[i,'Com E-USD']=sumEUSD
    pcData2018.at[i,'Error']=str(err)
    
## Produce different versions of the conversion factors

# Code to solve a bug - when comtrade units is 'weight in kilograms' sometimes the netweight column is blank
# => money-based conversion factor possible, but not currently calculated (about 20 - e.g. PRCCODE - 13201400 - woven fab..)

a=pcData2018[['Code','Com I-kg']]
b=pcData2018[['Code','Com I-QTY']][pcData2018['Com I-Unit']=='Weight in kilograms']
b.columns=['Code','Com I-QTY']
c=pd.merge(a,b,how='left',on='Code')
d=c['Com I-kg']
e=c['Com I-QTY']
d.update(e)
pcData2018['Com I-kg']=d


a=pcData2018[['Code','Com E-kg']]
b=pcData2018[['Code','Com E-QTY']][pcData2018['Com E-Unit']=='Weight in kilograms']
b.columns=['Code','Com E-QTY']
c=pd.merge(a,b,how='left',on='Code')
d=c['Com E-kg']
e=c['Com E-QTY']
d.update(e)
pcData2018['Com E-kg']=d


pcData2018['Unity factor kg']=''
pcData2018['Com Fac I']=pcData2018['Com I-kg'][pcData2018['Com I-QTY']!=0]//pcData2018['Com I-QTY'][pcData2018['Com I-QTY']!=0]
pcData2018['Com Fac E']=pcData2018['Com E-kg'][pcData2018['Com E-QTY']!=0]/pcData2018['Com E-QTY'][pcData2018['Com E-QTY']!=0]
pcData2018['Com ValFac I']=pcData2018['Com I-kg'][pcData2018['Com I-USD']!=0]/pcData2018['Com I-USD'][pcData2018['Com I-USD']!=0]
pcData2018['Com ValFac E']=pcData2018['Com E-kg'][pcData2018['Com E-USD']!=0]/pcData2018['Com E-USD'][pcData2018['Com E-USD']!=0]
pcData2018['Conversion factor error']=''
pcData2018['Selected factor']=''
pcData2018['Conv units']=''


for i, pcUnit in enumerate(pcData2018['PC unit']):
    err=[]

    #First, look for whether a value-based conversion factor is possible:
    
    avvalfac=np.nanmean([pcData2018.at[i,'Com ValFac I'],pcData2018.at[i,'Com ValFac E']])
    if np.isnan(avvalfac):
        err.append('Value-based factor not possible')
    else:
        pcData2018.at[i,'Selected factor']=avvalfac
        pcData2018.at[i,'Conv units']= 'kg / $'
        
        
    #Next, Look for wehther a comtrade netweight-QTY conversion factor makes sense
    
    try: 
        unitCorrsDict[pcUnit] 
    except:
        err.append('Prodcom unit not recognised')           
    else:       
        if pcData2018['Com I-Unit'][i]!=unitCorrsDict[pcUnit]:
            err.append('Prodcom and comtrade I unit inconsistent')
            pcData2018.at[i,'Com Fac I']='NaN'
            
        if pcData2018['Com E-Unit'][i]!=unitCorrsDict[pcUnit]:
            err.append('Prodcom and comtrade E unit inconsistent')
            pcData2018.at[i,'Com Fac E']='NaN'
      
        avfac=np.nanmean([pcData2018.at[i,'Com Fac I'],pcData2018.at[i,'Com Fac E']])
        if np.isnan(avfac):
            err.append('QTY factor not possible')
        else:
            pcData2018.at[i,'Selected factor']=avfac
            pcData2018.at[i,'Conv units']= 'kg / QTY'
       
      
     # Finally, look to see if the pc unit is kg - this takes priority, so do this last
      
    if pcUnit == 'kg':
        pcData2018.at[i,'Unity factor kg']=1.0
        pcData2018.at[i,'Selected factor']=1.0
        pcData2018.at[i,'Conv units']= 'kg / kg'
        
    else:
        err.append('Prodcom not in kg')
    
    pcData2018.at[i,'Conversion factor error']=err

# pcData2018.to_excel('../outputs/Prodcom-ComtradeCombined.xlsx',sheet_name='data')


## Evaluate mass half-flows for each Prodcom code

exch=1.18

pcData2018['P-kg']=''
pcData2018['I-kg']=''
pcData2018['E-kg']=''

pcData2018['P-kg - kg/kg']=pcData2018['P-QTY'][pcData2018['Conv units']=='kg / kg']
pcData2018['I-kg - kg/kg']=pcData2018['I-QTY'][pcData2018['Conv units']=='kg / kg']
pcData2018['E-kg - kg/kg']=pcData2018['E-QTY'][pcData2018['Conv units']=='kg / kg']

pcData2018['Mean ComFac']=pcData2018[['Com Fac I','Com Fac E']].mean(axis=1, skipna = True)
pcData2018['Mean ComValFac']=pcData2018[['Com ValFac I','Com ValFac E']].mean(axis=1, skipna = True)

pcData2018['P-kg - kg/QTY']=pcData2018['P-QTY']*pcData2018['Mean ComFac']
pcData2018['I-kg - kg/QTY']=pcData2018['I-QTY']*pcData2018['Com Fac I']
pcData2018['E-kg - kg/QTY']=pcData2018['E-QTY']*pcData2018['Com Fac E']

pcData2018['P-kg - kg/$']=pcData2018['P-EUR']*pcData2018['Mean ComValFac']*exch
pcData2018['I-kg - kg/$']=pcData2018['I-EUR']*pcData2018['Com ValFac I']*exch
pcData2018['E-kg - kg/$']=pcData2018['E-EUR']*pcData2018['Com ValFac E']*exch


#pcData2018['P-kg - kg/kg'] = pcData2018['P-QTY'][pcData2018['Conv units']=='kg / kg']
#pcData2018['P-kg - kg/QTY'] = pcData2018['P-QTY'][pcData2018['Conv units']=='kg / QTY']*pcData2018['Selected factor']
#pcData2018['P-kg - kg/$'] = pcData2018['P-EUR'][pcData2018['Conv units']=='kg / $']*pcData2018['Selected factor']*exch


for i, convunit in enumerate(pcData2018['Conv units']):

    if convunit=='kg / QTY' or convunit=='kg / kg':
        pcData2018.at[i,'P-kg']=pcData2018.at[i,'P-QTY']*pcData2018.at[i,'Selected factor']
        pcData2018.at[i,'I-kg']=pcData2018.at[i,'I-QTY']*pcData2018.at[i,'Selected factor']
        pcData2018.at[i,'E-kg']=pcData2018.at[i,'E-QTY']*pcData2018.at[i,'Selected factor']
    else:
        if convunit=='kg / $':
            pcData2018.at[i,'P-kg']=pcData2018.at[i,'P-EUR']*pcData2018.at[i,'Selected factor']*exch
            pcData2018.at[i,'I-kg']=pcData2018.at[i,'I-EUR']*pcData2018.at[i,'Selected factor']*exch
            pcData2018.at[i,'E-kg']=pcData2018.at[i,'E-EUR']*pcData2018.at[i,'Selected factor']*exch              

# pcData2018.to_excel('../outputs/Prodcom-ComtradeCombined.xlsx',sheet_name='data')


# ## Check a few things, and visualise the data a bit

# In[7]:


# Is there monetary data in Prodcom, but no conversion factor?
m=1000
a=pcData2018['Conv units']==''
b=pcData2018[['P-EUR','I-EUR','E-EUR']].sum(axis=1, skipna = True)>m
tf1=pd.concat([a,b],axis=1)
pcData2018['Warning - monetary data, no conv factor']= tf1.all(axis=1)

# Is there confidential data?
a=pcData2018['P-QTY'].isna()
b=pcData2018['I-QTY'].isna()
c=pcData2018['E-QTY'].isna()
tf2=pd.concat([a,b,c],axis=1)
pcData2018['Warning - confidential data']= tf2.any(axis=1)


# Save

# Add an index
# pcData2018.reset_index()
# pcData2018.index.name = "ID"

# XXX this should save directly into DataSources/... with the index=True
pcData2018.to_csv('outputs/Prodcom-ComtradeCombined.csv', index=False)
