Skip to content

Commit 9dd0343

Browse files
aunindadfaunindadf
committed
Data updated.
1 parent 08bb571 commit 9dd0343

8 files changed

Lines changed: 315 additions & 48 deletions

File tree

‎data/solr-config.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,12 @@
117117
},
118118
"C":
119119
{
120-
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND ",
120+
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
121121
"fieldDependency": "recipient_country_code"
122122
},
123123
"R":
124124
{
125-
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND ",
125+
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
126126
"fieldDependency": "recipient_region_code"
127127
},
128128
"O":

‎devtracker.rb‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
require "sinatra/cookies"
2323
require "cgi"
2424
require 'set'
25+
require 'uri'
2526

2627
#helpers path
2728
require_relative 'helpers/formatters.rb'
@@ -62,7 +63,7 @@
6263
# set :oipa_api_url, 'https://devtracker-entry.oipa.nl/api/'
6364
set :oipa_api_url, 'https://fcdo.iati.cloud/search/'
6465
# set :oipa_api_url, 'https://fcdo-direct-indexing.iati.cloud/search/'
65-
set :bind, '0.0.0.0' # Allows for vagrant pass-through whilst debugging
66+
#set :bind, '0.0.0.0' # Allows for vagrant pass-through whilst debugging
6667

6768
# Server Machine: set global settings to use varnish cache
6869
#set :oipa_api_url, 'http://127.0.0.1:6081/search/'
@@ -81,8 +82,8 @@
8182

8283
set :current_first_day_of_financial_year, first_day_of_financial_year(DateTime.now)
8384
set :current_last_day_of_financial_year, last_day_of_financial_year(DateTime.now)
84-
#set :goverment_department_ids, 'GB-GOV-25,GB-GOV-26,GB-GOV-15,GB-GOV-9,GB-GOV-6,GB-GOV-2,GB-GOV-1,GB-1,GB-GOV-3,GB-GOV-13,GB-GOV-7,GB-6,GB-10,GB-GOV-10,GB-9,GB-GOV-8,GB-GOV-5,GB-GOV-12,GB-COH-RC000346,GB-COH-03877777,GB-GOV-24'
85-
set :goverment_department_ids, 'GB-GOV-1,GB-1'
85+
set :goverment_department_ids, 'GB-GOV-25,GB-GOV-26,GB-GOV-15,GB-GOV-9,GB-GOV-6,GB-GOV-2,GB-GOV-1,GB-1,GB-GOV-3,GB-GOV-13,GB-GOV-7,GB-6,GB-10,GB-GOV-10,GB-9,GB-GOV-8,GB-GOV-5,GB-GOV-12,GB-COH-RC000346,GB-COH-03877777,GB-GOV-24'
86+
#set :goverment_department_ids, 'GB-GOV-1,GB-1'
8687
set :google_recaptcha_publicKey, ENV["GOOGLE_PUBLIC_KEY"]
8788
set :google_recaptcha_privateKey, ENV["GOOGLE_PRIVATE_KEY"]
8889

@@ -217,8 +218,9 @@ def getCacheData(fileName)
217218
end
218219
countryYearWiseBudgets = ''
219220
countrySectorGraphData = ''
220-
newtempActivityCount = 'activity?q=activity_status_code:2 AND hierarchy:1 AND participating_org_ref:GB-GOV-* AND reporting_org_ref:('+settings.goverment_department_ids.gsub(","," OR ")+') AND recipient_country_code:('+n+')&fl=sector_code,sector_percentage,sector_narrative,sector,recipient_country_code,recipient_country_name,recipient_country_percentage,recipient_country,recipient_region_code,recipient_region_name,recipient_region_percentage,recipient_region&rows=1'
221-
tempActivityCount = Oj.load(RestClient.get api_simple_log(settings.oipa_api_url + newtempActivityCount))['response']['numFound']
221+
newtempActivityCount = "activity?q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND reporting_org_ref:("+settings.goverment_department_ids.gsub(","," OR ")+") AND recipient_country_code:("+n+") AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z]&fl=sector_code,sector_percentage,sector_narrative,sector,recipient_country_code,recipient_country_name,recipient_country_percentage,recipient_country,recipient_region_code,recipient_region_name,recipient_region_percentage,recipient_region&rows=1"
222+
# puts settings.oipa_api_url + newtempActivityCount
223+
tempActivityCount = projectCounter(n)#Oj.load(RestClient.get api_simple_log(settings.oipa_api_url + newtempActivityCount))['response']['numFound']
222224
if n == 'UA2'
223225
tempActivityCount = 0
224226
end
@@ -355,7 +357,7 @@ def getCacheData(fileName)
355357
# Region summary page
356358
get '/regions/:region_code/?' do |n|
357359
n = sanitize_input(n,"p")
358-
region = get_region_detailsv2(n)
360+
region = get_region_detailsv3(n)
359361
settings.devtracker_page_title = 'Region '+region[:name]+' Summary Page'
360362
erb :'regions/region',
361363
:layout => :'layouts/layout',
@@ -372,7 +374,7 @@ def getCacheData(fileName)
372374
n = sanitize_input(n, "p")
373375
query = '('+n+')'
374376
filters = prepareFilters(query.to_s, 'R')
375-
response = solrResponse(query, 'AND activity_status_code:(2)', 'R', 0, '', '')
377+
response = solrResponse(query, "AND activity_status_code:(2)", 'R', 0, '', '')
376378
if(response['numFound'].to_i > 0)
377379
response = addTotalBudgetWithCurrency(response)
378380
end
@@ -804,7 +806,7 @@ def getCacheData(fileName)
804806
country_sector_data = ''
805807
getMaxBudget = ''
806808
if (!canLoadFromCache('country_sector_data'))
807-
storeCacheData(generateCountryDatav6(), 'country_sector_data')
809+
storeCacheData(generateCountryDatav7(), 'country_sector_data')
808810
getMainData = getCacheData('country_sector_data')
809811
map_data = getMainData['map_data']
810812
storeCacheData(map_data[1].sort_by{|k,val| val['budget']}.reverse!.first[1]['budget'], 'getMaxBudgetCountryLocation')

‎helpers/common_helpers.rb‎

Lines changed: 126 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1283,6 +1283,7 @@ def generateCountryDatav6()
12831283
page = page.to_i - 1
12841284
finalPage = page * count
12851285
######
1286+
puts newApiCall
12861287
pd = RestClient.get newApiCall
12871288
pd = JSON.parse(pd)
12881289
numOActivities = pd['response']['numFound'].to_i
@@ -1298,8 +1299,7 @@ def generateCountryDatav6()
12981299
pulledData.push(item)
12991300
end
13001301
end
1301-
end
1302-
puts ('Activity count: ' + pulledData.count.to_s)
1302+
end
13031303
projectDataHash = {}
13041304
##
13051305
fcdoCountryProjectTracker = {}
@@ -1333,7 +1333,7 @@ def generateCountryDatav6()
13331333
countryPercentage = 100
13341334
end
13351335
countryBudget = tempTotalBudget*countryPercentage/100
1336-
budgetTracker = budgetTracker - countryBudget
1336+
#budgetTracker = budgetTracker - countryBudget
13371337
if(projectDataHash.has_key?(c))
13381338
if(isNewProgramme)
13391339
projectDataHash[c]["projects"] = projectDataHash[c]["projects"] + 1
@@ -1346,14 +1346,133 @@ def generateCountryDatav6()
13461346
elsif c =='PS'
13471347
projectDataHash[c]["country"] = 'Occupied Palestinian Territories (OPT)'
13481348
else
1349-
projectDataHash[c]["country"] = begin get_country_code_name(c)['name'] rescue 'N/A' end# element.has_key?('recipient_country_name') ? element["recipient_country_name"][i] : 'N/A'
1349+
projectDataHash[c]["country"] = begin get_country_code_name(c)['name'] rescue 'N/A' end
13501350
end
13511351
projectDataHash[c]["id"] = c
1352-
if(isNewProgramme)
1353-
projectDataHash[c]["projects"] = 1
1352+
projectDataHash[c]["projects"] = 1
1353+
projectDataHash[c]["budget"] = countryBudget.round(2)
1354+
projectDataHash[c]["flag"] = '/images/flags/' + c.downcase + '.png'
1355+
end
1356+
end
1357+
end
1358+
finalOutput = Array.new
1359+
finalOutput.push(projectDataHash.to_s.gsub("[", "").gsub("]", "").gsub("=>",":").gsub("}}, {","},"))
1360+
finalOutput.push(projectDataHash)
1361+
output = {}
1362+
output['map_data'] = finalOutput
1363+
output
1364+
end
1365+
###############################
1366+
def escape_url_component(component_string)
1367+
URI.encode_www_form_component(component_string)
1368+
end
1369+
1370+
def hasSpecialCharacter(input_string)
1371+
# 1. Remove leading and trailing spaces
1372+
# The strip method returns a new string with leading and trailing whitespace removed.
1373+
trimmed_string = input_string.strip
1374+
1375+
# 2. Define the pattern of special characters to look for within the trimmed string.
1376+
# This regex matches:
1377+
# ' ' (a space)
1378+
# '/' (a forward slash)
1379+
# '&' (an ampersand)
1380+
# '\\' (a backslash - needs to be escaped in the regex pattern itself)
1381+
special_chars_regex = /[ \/&\\<?>]/
1382+
1383+
# 3. Check if the trimmed string contains any character matching the regex pattern.
1384+
# match? returns true if the pattern is found anywhere in the string, false otherwise.
1385+
if trimmed_string.match?(special_chars_regex)
1386+
return true
1387+
else
1388+
return false
1389+
end
1390+
end
1391+
1392+
def generateCountryDatav7()
1393+
count = 20
1394+
activityTracker = Set.new
1395+
newApiCall = settings.oipa_api_url + "activity?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND budget_value_gbp:* AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:*&fl=reporting_org_ref,recipient_country_percentage,activity_status_code,iati_identifier,budget.period-start.quarter,budget.period-end.quarter,recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,budget_value_gbp,recipient_country_name,hierarchy,related_activity_type,related_activity_ref&start=0&rows=#{count}"
1396+
##pagination stuff
1397+
page = 1
1398+
page = page.to_i - 1
1399+
finalPage = page * count
1400+
######
1401+
puts newApiCall
1402+
pd = RestClient.get newApiCall
1403+
pd = JSON.parse(pd)
1404+
numOActivities = pd['response']['numFound'].to_i
1405+
pulledData = pd['response']['docs']
1406+
if (numOActivities > count)
1407+
pages = (numOActivities.to_f/count).ceil
1408+
for p in 2..pages do
1409+
p = p - 1
1410+
finalPage = p * count
1411+
tempData = JSON.parse(RestClient.get settings.oipa_api_url + "activity?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND budget_value_gbp:* AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:*&fl=reporting_org_ref,recipient_country_percentage,activity_status_code,iati_identifier,budget.period-start.quarter,budget.period-end.quarter,recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,budget_value_gbp,recipient_country_name,hierarchy,related_activity_type,related_activity_ref&start=#{finalPage}&rows=#{count}")
1412+
tempData = tempData['response']['docs']
1413+
tempData.each do |item|
1414+
pulledData.push(item)
1415+
end
1416+
end
1417+
end
1418+
projectDataHash = {}
1419+
##
1420+
fcdoCountryProjectTracker = {}
1421+
countryDataHash = {}
1422+
##
1423+
pulledData.each do |element|
1424+
######New 2.0 version starts here#####
1425+
tempTotalBudget = 0
1426+
element['budget_value_gbp'].each_with_index do |data, index|
1427+
if(element['budget_period_start_iso_date'][index].to_datetime >= settings.current_first_day_of_financial_year && element['budget_period_end_iso_date'][index].to_datetime <= settings.current_last_day_of_financial_year)
1428+
tempTotalBudget = tempTotalBudget + data.to_f
1429+
end
1430+
end
1431+
## Process project budget and count now
1432+
# Get the parent identifier for this activity
1433+
# if element["hierarchy"].to_i != 1
1434+
# parentProgrammeID = element['related_activity_ref'][find_string_index(element['related_activity_type'],"1")]
1435+
# else
1436+
# parentProgrammeID = element['iati_identifier']
1437+
# end
1438+
# if activityTracker.include?(parentProgrammeID)
1439+
# isNewProgramme = false
1440+
# else
1441+
# isNewProgramme = true
1442+
# activityTracker.add(parentProgrammeID)
1443+
# end
1444+
element['recipient_country_code'].each_with_index do |c, i|
1445+
if element.has_key?('recipient_country_percentage')
1446+
countryPercentage = element['recipient_country_percentage'][i].to_f
1447+
else
1448+
countryPercentage = 100
1449+
end
1450+
countryBudget = tempTotalBudget*countryPercentage/100
1451+
#budgetTracker = budgetTracker - countryBudget
1452+
if(projectDataHash.has_key?(c))
1453+
# if(isNewProgramme)
1454+
# projectDataHash[c]["projects"] = projectDataHash[c]["projects"] + 1
1455+
# end
1456+
projectDataHash[c]["budget"] = (projectDataHash[c]["budget"] + countryBudget).round(2)
1457+
else
1458+
projectDataHash[c] = {}
1459+
if c =='FK'
1460+
projectDataHash[c]["country"] = 'Falkland Islands'
1461+
elsif c =='PS'
1462+
projectDataHash[c]["country"] = 'Occupied Palestinian Territories (OPT)'
13541463
else
1355-
projectDataHash[c]["projects"] = 0
1464+
projectDataHash[c]["country"] = begin get_country_code_name(c)['name'] rescue 'N/A' end
1465+
end
1466+
projectDataHash[c]["id"] = c
1467+
call = settings.oipa_api_url + "activity?q=activity_status_code:2 AND hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:#{c}&fl=iati_identifier&rows=0"
1468+
pd = RestClient.get call
1469+
pd = JSON.parse(pd)
1470+
numOProgs = pd['response']['numFound'].to_i
1471+
if c == 'BD'
1472+
puts call
1473+
puts numOProgs
13561474
end
1475+
projectDataHash[c]["projects"] = numOProgs
13571476
projectDataHash[c]["budget"] = countryBudget.round(2)
13581477
projectDataHash[c]["flag"] = '/images/flags/' + c.downcase + '.png'
13591478
end

‎helpers/country_helpers.rb‎

Lines changed: 34 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ def get_country_results(countryCode)
199199
end
200200

201201
def get_country_or_regionv2(projectId)
202+
projectId = escape_url_component(projectId)
202203
countryOrRegionAPI = RestClient.get api_simple_log(settings.oipa_api_url + "/activity/?q=iati_identifier:#{projectId}&fl=iati_identifier,recipient_country_name,recipient_country_code,recipient_region_code,recipient_region_name,recipient_country_percentage,recipient_region_percentage")
203204
countryOrRegionData = JSON.parse(countryOrRegionAPI)['response']['docs'].first
204205
countries = []
@@ -335,9 +336,33 @@ def budgetBarGraphData(apiLink)
335336
data
336337
end
337338

339+
def projectCounter(countryCode)
340+
activityTracker = Set.new
341+
newApiCall = settings.oipa_api_url + "activity?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND budget_value_gbp:* AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND recipient_country_code:"+countryCode+"&fl=iati_identifier,budget_period_start_iso_date,budget_period_end_iso_date,hierarchy,related_activity_type,related_activity_ref&start=0&rows=7000"
342+
puts newApiCall
343+
pd = RestClient.get newApiCall
344+
pd = JSON.parse(pd)
345+
pulledData = pd['response']['docs']
346+
pulledData.each do |element|
347+
if element["hierarchy"].to_i != 1
348+
parentProgrammeID = element['related_activity_ref'][find_string_index(element['related_activity_type'],"1")]
349+
else
350+
parentProgrammeID = element['iati_identifier']
351+
end
352+
if activityTracker.include?(parentProgrammeID)
353+
isNewProgramme = false
354+
else
355+
isNewProgramme = true
356+
activityTracker.add(parentProgrammeID)
357+
end
358+
end
359+
puts "activity count: #{activityTracker.size()}"
360+
activityTracker.size()
361+
end
362+
338363
def budgetBarGraphDataDv3(countryCode)
339364
#Process budgets
340-
apiData = RestClient.get api_simple_log(settings.oipa_api_url + "activity/?q=budget_value_gbp:* AND participating_org_ref:GB-GOV-* AND recipient_country_code:#{countryCode} AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")})&fl=related_budget_period_end_iso_date,recipient_country_percentage,recipient_country_code,related_budget_period_start_iso_date,related_budget_value,related_budget_period_start_quarter,reporting_org_narrative,reporting_org_ref,budget.period-start.quarter,transaction.transaction-date.quarter,transaction_type,transaction_date_iso_date,transaction_value,budget_value_gbp,budget_period_start_iso_date,budget_period_end_iso_date,budget_value&start=0&rows=10000")
365+
apiData = RestClient.get api_simple_log(settings.oipa_api_url + "activity/?q=budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND budget_value_gbp:* AND participating_org_ref:GB-GOV-* AND recipient_country_code:#{countryCode} AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")})&fl=related_budget_period_end_iso_date,recipient_country_percentage,recipient_country_code,related_budget_period_start_iso_date,related_budget_value,related_budget_period_start_quarter,reporting_org_narrative,reporting_org_ref,budget.period-start.quarter,transaction.transaction-date.quarter,transaction_type,transaction_date_iso_date,transaction_value,budget_value_gbp,budget_period_start_iso_date,budget_period_end_iso_date,budget_value&start=0&rows=10000")
341366
apiData = JSON.parse(apiData)['response']['docs']
342367
fyTracker = []
343368
repOrgs = {}
@@ -373,13 +398,13 @@ def budgetBarGraphDataDv3(countryCode)
373398
fyTracker.push(fy)
374399
end
375400
end
376-
tempT = {}
377-
tempT['rep-org'] = element['reporting_org_ref']
378-
tempT['startDate'] = element['budget_period_start_iso_date'][index]
379-
tempT['endDate'] = element['budget_period_end_iso_date'][index]
380-
tempT['countryPercentage'] = countryPercentage
381-
tempT['cBudget'] = data.to_f*countryPercentage/100
382-
tracker.append(tempT)
401+
# tempT = {}
402+
# tempT['rep-org'] = element['reporting_org_ref']
403+
# tempT['startDate'] = element['budget_period_start_iso_date'][index]
404+
# tempT['endDate'] = element['budget_period_end_iso_date'][index]
405+
# tempT['countryPercentage'] = countryPercentage
406+
# tempT['cBudget'] = data.to_f*countryPercentage/100
407+
# tracker.append(tempT)
383408
end
384409
end
385410
# if(element['reporting_org_ref'].to_s == 'GB-GOV-1' || element['reporting_org_ref'].to_s == 'GB-1')
@@ -639,7 +664,7 @@ def get_country_all_projects_rss(countryCode)
639664
end
640665

641666
def total_country_budget_locationv2
642-
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND recipient_country_code:*&fl=recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,budget_value_gbp,recipient_country_name&rows=50000"
667+
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z] AND recipient_country_code:* AND budget_value_gbp:* AND participating_org_ref:(GB-GOV-* OR GB-COH-*)&fl=recipient_country_code,budget_period_start_iso_date,budget_period_end_iso_date,budget_value_gbp,recipient_country_name&rows=50000"
643668
pulledData = RestClient.get newApiCall
644669
pulledData = JSON.parse(pulledData)['response']['docs']
645670
totalBudget = 0
@@ -656,19 +681,6 @@ def total_country_budget_locationv2
656681
end
657682
totalAmount = (format_million_stg totalBudget.to_f).to_s.gsub("&pound;","")
658683
totalAmount
659-
########
660-
#oipa 3.1
661-
# totalCountryBudgetLocation = RestClient.get api_simple_log(settings.oipa_api_url + "budgets/aggregations/?reporting_organisation_identifier=#{settings.goverment_department_ids}&group_by=recipient_country&aggregations=value&budget_period_start=#{firstDayOfFinYear}&budget_period_end=#{lastDayOfFinYear}&format=json&order_by=-value")
662-
# totalCountryBudgetLocation = JSON.parse(totalCountryBudgetLocation)
663-
# totalAmount = 0.0
664-
# totalCountryBudgetLocation['results'].each do |countryBudgets|
665-
# #oipa 2.2
666-
# #totalAmount = totalAmount + countryBudgets['budget'].to_f
667-
# #oipa 3.1
668-
# totalAmount = totalAmount + countryBudgets['value'].to_f
669-
# end
670-
# totalAmount = (format_million_stg totalAmount.to_f).to_s.gsub("&pound;","")
671-
# totalAmount
672684
end
673685

674686
def total_country_budget_location

0 commit comments

Comments
 (0)