Skip to content

Commit f3f17b9

Browse files
aunindadfaunindadf
committed
Updated a few things and fixed some calculation bugs.
1 parent cb40719 commit f3f17b9

7 files changed

Lines changed: 258 additions & 80 deletions

File tree

‎data/dfidRegions.json‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,5 +68,75 @@
6868
"description":"",
6969
"type":"Global",
7070
"url":"https://iati.fcdo.gov.uk/iati_files/Region/DFID-Developing-countries-unspecified-998-1.xml"
71+
},
72+
{
73+
"code":"1031",
74+
"name":"Caribbean",
75+
"description":"",
76+
"type":"Regional",
77+
"url":""
78+
},
79+
{
80+
"code":"619",
81+
"name":"Central Asia",
82+
"description":"",
83+
"type":"Regional",
84+
"url":""
85+
},
86+
{
87+
"code":"1027",
88+
"name":"Eastern Africa",
89+
"description":"",
90+
"type":"Regional",
91+
"url":""
92+
},
93+
{
94+
"code":"789",
95+
"name":"Far East Asia",
96+
"description":"",
97+
"type":"Regional",
98+
"url":""
99+
},
100+
{
101+
"code":"889",
102+
"name":"Oceania",
103+
"description":"",
104+
"type":"Regional",
105+
"url":""
106+
},
107+
{
108+
"code":"689",
109+
"name":"South & Central Asia",
110+
"description":"",
111+
"type":"Regional",
112+
"url":""
113+
},
114+
{
115+
"code":"489",
116+
"name":"South America",
117+
"description":"",
118+
"type":"Regional",
119+
"url":""
120+
},
121+
{
122+
"code":"1029",
123+
"name":"Southern Africa",
124+
"description":"",
125+
"type":"Regional",
126+
"url":""
127+
},
128+
{
129+
"code":"1030",
130+
"name":"Western Africa",
131+
"description":"",
132+
"type":"Regional",
133+
"url":""
134+
},
135+
{
136+
"code":"498",
137+
"name":"America",
138+
"description":"",
139+
"type":"Regional",
140+
"url":""
71141
}
72142
]

‎devtracker.rb‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@
6161
# Developer Machine: set global settings
6262
#set :oipa_api_url, 'https://fcdo-direct-indexing.iati.cloud/search/'#'https://fcdo.iati.cloud/search/'#'https://fcdo-direct-indexing.iati.cloud/search/'#'https://devtracker.fcdo.gov.uk/api/'
6363
# set :oipa_api_url, 'https://devtracker-entry.oipa.nl/api/'
64-
set :oipa_api_url, 'https://fcdo.iati.cloud/search/'
64+
#set :oipa_api_url, 'https://fcdo.iati.cloud/search/'
6565
# set :oipa_api_url, 'https://fcdo-direct-indexing.iati.cloud/search/'
6666
#set :bind, '0.0.0.0' # Allows for vagrant pass-through whilst debugging
6767

6868
# Server Machine: set global settings to use varnish cache
69-
#set :oipa_api_url, 'http://127.0.0.1:6081/search/'
69+
set :oipa_api_url, 'http://127.0.0.1:6081/search/'
7070
set :prod_api_url, 'https://fcdo.iati.cloud'
7171
set :dev_api_url, 'https://fcdo-staging.iati.cloud'
7272

@@ -82,8 +82,8 @@
8282

8383
set :current_first_day_of_financial_year, first_day_of_financial_year(DateTime.now)
8484
set :current_last_day_of_financial_year, last_day_of_financial_year(DateTime.now)
85-
set :goverment_department_ids, 'GB-GOV-25,GB-GOV-26,GB-GOV-15,GB-GOV-9,GB-GOV-6,GB-GOV-2,GB-GOV-1,GB-1,GB-GOV-3,GB-GOV-13,GB-GOV-7,GB-6,GB-10,GB-GOV-10,GB-9,GB-GOV-8,GB-GOV-5,GB-GOV-12,GB-COH-RC000346,GB-COH-03877777,GB-GOV-24'
86-
#set :goverment_department_ids, 'GB-GOV-1,GB-1'
85+
set :goverment_department_ids, 'GB-GOV-11,GB-GOV-14,GB-GOV-27,GB-GOV-53,GB-GOV-25,GB-GOV-26,GB-GOV-15,GB-GOV-9,GB-GOV-6,GB-GOV-2,GB-GOV-1,GB-1,GB-GOV-3,GB-GOV-13,GB-GOV-7,GB-6,GB-10,GB-GOV-10,GB-9,GB-GOV-8,GB-GOV-5,GB-GOV-12,GB-COH-RC000346,GB-COH-03877777,GB-GOV-24'
86+
#set :goverment_department_ids, 'GB-GOV-26,GB-GOV-1,GB-GOV-13,GB-GOV-10'
8787
set :google_recaptcha_publicKey, ENV["GOOGLE_PUBLIC_KEY"]
8888
set :google_recaptcha_privateKey, ENV["GOOGLE_PRIVATE_KEY"]
8989

@@ -130,12 +130,14 @@
130130
## calculate the budget amount against that perentage and add it to
131131
## the high level related sector code
132132
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z]&fl=iati_identifier,budget_value,recipient_country_code,recipient_region_code,budget_period_start_iso_date,budget_period_end_iso_date,sector_code,sector_percentage,&rows=50000"
133+
133134
pulledData = RestClient.get newApiCall
134135
storeCacheData(high_level_sector_listv2(pulledData, 'top_five_sectors'), 'what_we_do')
135136
what_we_do = getCacheData('what_we_do')
136137
else
137138
what_we_do = getCacheData('what_we_do')
138139
end
140+
139141
whatWeDoTotal = what_we_do.first['budget']
140142
top5countries = top5countries.select{|i| i.has_key?('budget')}
141143
top5countries = top5countries.sort_by{|val| -val['budget'].to_f}
@@ -897,6 +899,19 @@ def getCacheData(fileName)
897899
activityStatuses = 'AND activity_status_code:(2)'
898900
filters = prepareFilters(query.to_s, 'F')
899901
response = solrResponse(query, activityStatuses, 'F', 0, '', '')
902+
##Clean any unnecessary activity
903+
## TODO
904+
elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
905+
if elist.length > 0
906+
tempResponseList = response['response']['docs']
907+
tempResponseList.each do |data|
908+
if elist.include?(data['iati_identifier'])
909+
tempResponseList.delete(data)
910+
end
911+
end
912+
response['response']['docs'] = tempResponseList
913+
end
914+
##
900915
if(response['response']['numFound'].to_i > 0)
901916
response['response'] = addTotalBudgetWithCurrency(response['response'])
902917
response = addHighlightingToFTSTerms(response)
@@ -951,6 +966,19 @@ def getCacheData(fileName)
951966
filters = prepareFilters(query.to_s, 'F')
952967
response = solrResponse(query, activityStatuses, 'F', 0, '', '')
953968
if(response['response']['numFound'].to_i > 0)
969+
##Clean any unnecessary activity
970+
## TODO
971+
elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
972+
if elist.length > 0
973+
tempResponseList = response['response']['docs']
974+
tempResponseList.each do |data|
975+
if elist.include?(data['iati_identifier'])
976+
tempResponseList.delete(data)
977+
end
978+
end
979+
response['response']['docs'] = tempResponseList
980+
end
981+
##
954982
response['response'] = addTotalBudgetWithCurrency(response['response'])
955983
response = addHighlightingToFTSTerms(response)
956984
end

‎helpers/common_helpers.rb‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,7 +1422,6 @@ def generateCountryDatav7()
14221422
page = page.to_i - 1
14231423
finalPage = page * count
14241424
######
1425-
puts newApiCall
14261425
pd = RestClient.get newApiCall
14271426
pd = JSON.parse(pd)
14281427
numOActivities = pd['response']['numFound'].to_i
@@ -1446,6 +1445,12 @@ def generateCountryDatav7()
14461445
##
14471446
pulledData.each do |element|
14481447
######New 2.0 version starts here#####
1448+
totalRegionPercentage = 0
1449+
if element.has_key?('recipient_region_percentage')
1450+
element['recipient_region_percentage'].each do |p|
1451+
totalRegionPercentage = totalRegionPercentage + p
1452+
end
1453+
end
14491454
tempTotalBudget = 0
14501455
element['budget_value_gbp'].each_with_index do |data, index|
14511456
if(element['budget_period_start_iso_date'][index].to_datetime >= settings.current_first_day_of_financial_year && element['budget_period_end_iso_date'][index].to_datetime <= settings.current_last_day_of_financial_year)
@@ -1480,7 +1485,7 @@ def generateCountryDatav7()
14801485
if element.has_key?('recipient_country_percentage')
14811486
countryPercentage = element['recipient_country_percentage'][i].to_f
14821487
else
1483-
countryPercentage = 100
1488+
countryPercentage = ((100-totalRegionPercentage)/element['recipient_country_code'].length).round(2)
14841489
end
14851490
countryBudget = tempTotalBudget*countryPercentage/100
14861491
totalB = totalB + countryBudget
@@ -1518,8 +1523,6 @@ def generateCountryDatav7()
15181523
finalOutput.push(projectDataHash.to_s.gsub("[", "").gsub("]", "").gsub("=>",":").gsub("}}, {","},"))
15191524
finalOutput.push(projectDataHash)
15201525
output = {}
1521-
puts 'Total budget: ' + totalB.to_s
1522-
puts 'Total budget no country filter: ' + totalB_s.to_s
15231526
output['map_data'] = finalOutput
15241527
output
15251528
end

‎helpers/country_helpers.rb‎

Lines changed: 23 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -88,25 +88,13 @@ def get_top_5_countriesv2()
8888
end
8989
newTop5countriesBudget = {}
9090
pulledData.each do |element|
91+
totalRegionPercentage = 0
92+
if element.has_key?('recipient_region_percentage')
93+
element['recipient_region_percentage'].each do |p|
94+
totalRegionPercentage = totalRegionPercentage + p
95+
end
96+
end
9197
tempTotalBudget = 0
92-
## Process total budget value first
93-
# if(element['reporting_org_ref'].to_s == 'GB-GOV-1')
94-
# if element.has_key?('related_budget_value')
95-
# element['related_budget_value'].each_with_index do |data, index|
96-
# if(element['related_budget_period_start_iso_date'][index].to_datetime >= firstDayOfFinYear && element['related_budget_period_end_iso_date'][index].to_datetime <= lastDayOfFinYear)
97-
# tempTotalBudget = tempTotalBudget + data.to_f
98-
# end
99-
# end
100-
# end
101-
# else
102-
# if element.has_key?('budget_value')
103-
# element['budget_value'].each_with_index do |data, index|
104-
# if(element['budget_period_start_iso_date'][index].to_datetime >= firstDayOfFinYear && element['budget_period_end_iso_date'][index].to_datetime <= lastDayOfFinYear)
105-
# tempTotalBudget = tempTotalBudget + data.to_f
106-
# end
107-
# end
108-
# end
109-
# end
11098
if element.has_key?('budget_value_gbp')
11199
element['budget_value_gbp'].each_with_index do |data, index|
112100
if(element['budget_period_start_iso_date'][index].to_datetime >= settings.current_first_day_of_financial_year && element['budget_period_end_iso_date'][index].to_datetime <= settings.current_last_day_of_financial_year)
@@ -116,7 +104,7 @@ def get_top_5_countriesv2()
116104
end
117105
if element.has_key?('recipient_country_code')
118106
element['recipient_country_code'].each_with_index do |cd, i|
119-
countryPercentage = element.has_key?('recipient_country_percentage') ? element['recipient_country_percentage'][i].to_f : 100
107+
countryPercentage = element.has_key?('recipient_country_percentage') ? element['recipient_country_percentage'][i].to_f : ((100-totalRegionPercentage)/element['recipient_country_code'].length).round(2)
120108
countryBudget = tempTotalBudget*countryPercentage/100
121109
if newTop5countriesBudget.has_key?(cd)
122110
newTop5countriesBudget[cd]['budget'] = (newTop5countriesBudget[cd]['budget'] + countryBudget).round(2)
@@ -152,11 +140,17 @@ def get_country_detailsv2(countryCode)
152140
countriesInfo = Oj.load(File.read('data/countries.json'))
153141
country = countriesInfo.select {|country| country['code'] == countryCode}.first
154142
## new api call
155-
newApiCall = settings.oipa_api_url + "activity?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:#{countryCode}&fl=budget_value_gbp,reporting_org_ref,recipient_country_percentage,activity_status_code,iati_identifier,budget.period-start.quarter,budget.period-end.quarter,recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,recipient_country_name,sector_code,sector_percentage,hierarchy,related_activity_type,related_activity_ref&rows=50000"
143+
newApiCall = settings.oipa_api_url + "activity?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:#{countryCode}&fl=recipient_region_percentage,budget_value_gbp,reporting_org_ref,recipient_country_percentage,activity_status_code,iati_identifier,budget.period-start.quarter,budget.period-end.quarter,recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,recipient_country_name,sector_code,sector_percentage,hierarchy,related_activity_type,related_activity_ref&rows=50000"
156144
pulledData = RestClient.get newApiCall
157145
pulledData = JSON.parse(pulledData)['response']['docs']
158146
countryTotalBudget = 0
159147
pulledData.each do |element|
148+
totalRegionPercentage = 0
149+
if element.has_key?('recipient_region_percentage')
150+
element['recipient_region_percentage'].each do |p|
151+
totalRegionPercentage = totalRegionPercentage + p
152+
end
153+
end
160154
tempTotalBudget = 0
161155
element['budget_value_gbp'].each_with_index do |data, index|
162156
if(element['budget_period_start_iso_date'][index].to_datetime >= settings.current_first_day_of_financial_year && element['budget_period_end_iso_date'][index].to_datetime <= settings.current_last_day_of_financial_year)
@@ -166,14 +160,16 @@ def get_country_detailsv2(countryCode)
166160
if element.has_key?('recipient_country_code')
167161
element['recipient_country_code'].each_with_index do |c, i|
168162
if c.to_s == countryCode
169-
countryPercentage = element.has_key?('recipient_country_percentage') ? element['recipient_country_percentage'][i].to_f : 100
163+
countryPercentage = element.has_key?('recipient_country_percentage') ? element['recipient_country_percentage'][i].to_f : ((100-totalRegionPercentage)/element['recipient_country_code'].length).round(2)
170164
countryBudget = tempTotalBudget*countryPercentage/100
171165
countryTotalBudget = countryTotalBudget + countryBudget.round(2)
172166
break
173167
end
174168
end
175169
end
176170
end
171+
puts 'xxx---xxx'
172+
puts countryTotalBudget
177173
returnObject = {
178174
:code => country['code'],
179175
:name => country['name'],
@@ -339,13 +335,16 @@ def budgetBarGraphData(apiLink)
339335
def projectCounter(countryCode)
340336
activityTracker = Set.new
341337
newApiCall = settings.oipa_api_url + "activity?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND budget_value_gbp:* AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:"+countryCode+"&fl=iati_identifier,budget_period_start_iso_date,budget_period_end_iso_date,hierarchy,related_activity_type,related_activity_ref&start=0&rows=7000"
342-
puts newApiCall
343338
pd = RestClient.get newApiCall
344339
pd = JSON.parse(pd)
345340
pulledData = pd['response']['docs']
346341
pulledData.each do |element|
347342
if element["hierarchy"].to_i != 1
348-
parentProgrammeID = element['related_activity_ref'][find_string_index(element['related_activity_type'],"1")]
343+
begin
344+
parentProgrammeID = element['related_activity_ref'][find_string_index(element['related_activity_type'],"1")]
345+
rescue
346+
puts element['iati_identifier']
347+
end
349348
else
350349
parentProgrammeID = element['iati_identifier']
351350
end
@@ -356,7 +355,6 @@ def projectCounter(countryCode)
356355
activityTracker.add(parentProgrammeID)
357356
end
358357
end
359-
puts "activity count: #{activityTracker.size()}"
360358
activityTracker.size()
361359
end
362360

@@ -663,7 +661,7 @@ def get_country_all_projects_rss(countryCode)
663661
end
664662

665663
def total_country_budget_locationv2
666-
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND recipient_country_code:* AND budget_value_gbp:* AND participating_org_ref:(GB-GOV-* OR GB-COH-*)&fl=recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,budget_value_gbp,recipient_country_name,recipient_region_percentage&rows=50000"
664+
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:(GB-GOV-* OR GB-COH-*) AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND recipient_country_code:* AND budget_value_gbp:*&fl=recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,budget_value_gbp,recipient_country_name,recipient_region_percentage&rows=50000"
667665
pulledData = RestClient.get newApiCall
668666
pulledData = JSON.parse(pulledData)['response']['docs']
669667
totalBudget = 0

‎helpers/project_helpers.rb‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ def check_if_project_exists(projectId)
3232
break
3333
end
3434
end
35+
elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
36+
if elist.length > 0
37+
if elist.include?(projectId.to_s)
38+
isGovOrgPresent = false
39+
end
40+
end
3541
if !isGovOrgPresent
3642
halt 404, "Activity not found"
3743
end

0 commit comments

Comments
 (0)