Skip to content

Commit a15a9a3

Browse files
aunindadfaunindadf
authored
Merge pull request #639 from DFID/test-new-api-nov-25
Test new api nov 25
2 parents 55d685f + 086490b commit a15a9a3

4 files changed

Lines changed: 229 additions & 69 deletions

File tree

‎data/solr-config.json‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
"document_link_title_narrative_text",
1414
"transaction",
1515
"reporting_org_narrative",
16-
"participating_org_narrative"
16+
"participating_org_narrative",
17+
"related_activity_context"
1718
],
1819
"SortingOptions":
1920
[
@@ -107,11 +108,86 @@
107108
"isFieldTypeString": "1"
108109
}
109110
],
111+
"Filters_f":
112+
[
113+
{
114+
"title": "Activity Status",
115+
"field": "activity_status_code",
116+
"url": "search/activity/?facet=on&facet.field=activity_status_code&facet.limit=-1&facet.mincount=1&q.op=AND&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*) ",
117+
"mappingFile": "data/mappingSolrFilters/activity_status.json",
118+
"defaultValue": "2",
119+
"isFieldTypeString": "0"
120+
},
121+
{
122+
"title": "Government Department(s)",
123+
"field": "reporting_org_ref",
124+
"url": "search/activity/?facet=on&facet.field=reporting_org_ref&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*) ",
125+
"mappingFile": "data/custom-codes/ogds.json",
126+
"defaultValue": "",
127+
"isFieldTypeString": "1"
128+
},
129+
{
130+
"title": "Tags",
131+
"field": "tag_code",
132+
"url": "search/activity/?facet=on&facet.field=tag_code&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*) ",
133+
"mappingFile": "data/custom-codes/policy-priorities.json",
134+
"defaultValue": "",
135+
"isFieldTypeString": "1"
136+
},
137+
{
138+
"title": "Sectors",
139+
"field": "sector_code",
140+
"url": "search/activity/?facet=on&facet.field=sector_code&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*) ",
141+
"mappingFile": "sectorHierarchies.json",
142+
"defaultValue": "",
143+
"isFieldTypeString": "0"
144+
},
145+
{
146+
"title": "Participating Orgs",
147+
"field": "participating_org_ref",
148+
"url": "search/activity/?facet=on&facet.field=participating_org_ref&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*)&fq=participating_org_role:4 ",
149+
"mappingFile": "iati_publishers_list.json",
150+
"defaultValue": "",
151+
"isFieldTypeString": "1"
152+
},
153+
{
154+
"title": "Activity Dates",
155+
"field": "activity_date_iso_date",
156+
"url": "search/activity/?facet=on&facet.field=activity_date_iso_date&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*)",
157+
"mappingFile": "",
158+
"defaultValue": "",
159+
"isFieldTypeString": "1"
160+
},
161+
{
162+
"title": "Benefiting Countries",
163+
"field": "recipient_country_code",
164+
"url": "search/activity/?facet=on&facet.field=recipient_country_code&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*)",
165+
"mappingFile": "data/country-codes/country_codes.json",
166+
"defaultValue": "",
167+
"isFieldTypeString": "1"
168+
},
169+
{
170+
"title": "Benefiting Regions",
171+
"field": "recipient_region_code",
172+
"url": "search/activity/?facet=on&facet.field=recipient_region_code&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*)",
173+
"mappingFile": "data/regionCodes.json",
174+
"defaultValue": "",
175+
"isFieldTypeString": "1"
176+
},
177+
{
178+
"title": "Document Categories",
179+
"field": "document_link_category_code",
180+
"url": "search/activity/?facet=on&facet.field=document_link_category_code&facet.limit=-1&facet.mincount=1&fq=hierarchy:1&fq=participating_org_ref:(GB-GOV-* OR GB-COH-*)",
181+
"mappingFile": "data/custom-codes/DocumentCategory.json",
182+
"defaultValue": "",
183+
"isFieldTypeString": "1"
184+
}
185+
],
110186
"QueryCategories":
111187
{
112188
"F":
113189
{
114-
"url": "/search/activity/?hl=on&hl.requireFieldMatch=true&hl.fl=title_narrative,title_narrative_first,description_narrative,iati_identifier,transaction_description_narrative,reporting_org_ref,recipient_country_name,document_link_title_narrative_text,transaction,reporting_org_narrative,participating_org_narrative&q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND ",
190+
"url": "/search/activity/?q.op=AND&sort=if(termfreq(reporting_org_ref,'GB-GOV-1'),1,if(termfreq(reporting_org_ref,'GB-GOV-2'),2,99)) asc, score desc&hl=on&hl.requireFieldMatch=true&hl.fl=title_narrative,title_narrative_first,description_narrative,iati_identifier,transaction_description_narrative,reporting_org_ref,recipient_country_name,document_link_title_narrative_text,transaction,reporting_org_narrative,participating_org_narrative&fq=hierarchy:1&fq=participating_org_ref:GB-GOV-*",
115191
"fieldDependency": "",
116192
"comments": "This search category is related to the free text search of DevTracker"
117193
},

‎devtracker.rb‎

Lines changed: 120 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,14 @@
102102
top5countries = ''
103103
Benchmark.bm(7) do |x|
104104
x.report("Loading Time: ") {
105-
if (!canLoadFromCache('top5Countries'))
106-
storeCacheData(get_top_5_countriesv2(), 'top5Countries')
107-
top5countries = getCacheData('top5Countries')
108-
else
109-
top5countries = getCacheData('top5Countries')
110-
end
105+
# if (!canLoadFromCache('top5Countries'))
106+
# storeCacheData(get_top_5_countriesv2(), 'top5Countries')
107+
# top5countries = getCacheData('top5Countries')
108+
# else
109+
# top5countries = getCacheData('top5Countries')
110+
# end
111+
#storeCacheData(get_top_5_countriesv2(), 'top5Countries')
112+
top5countries = get_top_5_countriesv2() #getCacheData('top5Countries')
111113
#oipa v3.1
112114
}
113115
end
@@ -118,25 +120,28 @@
118120
# Example of setting up a cookie from server end
119121
# response.set_cookie('my_cookie', 'value_of_cookie')
120122
what_we_do = ''
121-
if (!canLoadFromCache('what_we_do'))
122-
## logic
123-
## get budget data, then check if they fall within date range
124-
## do a summation of the budget amount within date range
125-
## pick dac5 sector code and then get the underlying high level sector code
126-
## if the sector is present, add the new budget info to this sector
127-
## Else create a new ref to the high level sector code and add
128-
## budget value starting from 0
129-
## pick budget percentage for each dac5 sector code2
130-
## calculate the budget amount against that perentage and add it to
131-
## the high level related sector code
132-
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z]&fl=iati_identifier,budget_value,recipient_country_code,recipient_region_code,budget_period_start_iso_date,budget_period_end_iso_date,sector_code,sector_percentage,&rows=50000"
123+
newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z]&fl=iati_identifier,budget_value,recipient_country_code,recipient_region_code,budget_period_start_iso_date,budget_period_end_iso_date,sector_code,sector_percentage,&rows=50000"
124+
pulledData = RestClient.get newApiCall
125+
what_we_do = high_level_sector_listv2(pulledData, 'top_five_sectors')
126+
# if (!canLoadFromCache('what_we_do'))
127+
# ## logic
128+
# ## get budget data, then check if they fall within date range
129+
# ## do a summation of the budget amount within date range
130+
# ## pick dac5 sector code and then get the underlying high level sector code
131+
# ## if the sector is present, add the new budget info to this sector
132+
# ## Else create a new ref to the high level sector code and add
133+
# ## budget value starting from 0
134+
# ## pick budget percentage for each dac5 sector code2
135+
# ## calculate the budget amount against that perentage and add it to
136+
# ## the high level related sector code
137+
# newApiCall = settings.oipa_api_url + "activity?q=participating_org_ref:GB-GOV-* AND reporting_org_ref:(#{settings.goverment_department_ids.gsub(","," OR ")}) AND budget_period_start_iso_date:[#{settings.current_first_day_of_financial_year}T00:00:00Z TO *] AND budget_period_end_iso_date:[* TO #{settings.current_last_day_of_financial_year}T00:00:00Z]&fl=iati_identifier,budget_value,recipient_country_code,recipient_region_code,budget_period_start_iso_date,budget_period_end_iso_date,sector_code,sector_percentage,&rows=50000"
133138

134-
pulledData = RestClient.get newApiCall
135-
storeCacheData(high_level_sector_listv2(pulledData, 'top_five_sectors'), 'what_we_do')
136-
what_we_do = getCacheData('what_we_do')
137-
else
138-
what_we_do = getCacheData('what_we_do')
139-
end
139+
# pulledData = RestClient.get newApiCall
140+
# storeCacheData(high_level_sector_listv2(pulledData, 'top_five_sectors'), 'what_we_do')
141+
# what_we_do = getCacheData('what_we_do')
142+
# else
143+
# what_we_do = getCacheData('what_we_do')
144+
# end
140145

141146
whatWeDoTotal = what_we_do.first['budget']
142147
top5countries = top5countries.select{|i| i.has_key?('budget')}
@@ -901,16 +906,16 @@ def getCacheData(fileName)
901906
response = solrResponse(query, activityStatuses, 'F', 0, '', '')
902907
##Clean any unnecessary activity
903908
## TODO
904-
elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
905-
if elist.length > 0
906-
tempResponseList = response['response']['docs']
907-
tempResponseList.each do |data|
908-
if elist.include?(data['iati_identifier'])
909-
tempResponseList.delete(data)
910-
end
911-
end
912-
response['response']['docs'] = tempResponseList
913-
end
909+
# elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
910+
# if elist.length > 0
911+
# tempResponseList = response['response']['docs']
912+
# tempResponseList.each do |data|
913+
# if elist.include?(data['iati_identifier'])
914+
# tempResponseList.delete(data)
915+
# end
916+
# end
917+
# response['response']['docs'] = tempResponseList
918+
# end
914919
##
915920
if(response['response']['numFound'].to_i > 0)
916921
response['response'] = addTotalBudgetWithCurrency(response['response'])
@@ -948,21 +953,21 @@ def getCacheData(fileName)
948953
query = sanitize_input(params['query'],"newId")
949954
isIncludeClosedProjects = sanitize_input(params['includeClosedProject'],"newId")
950955
if(isIncludeClosedProjects.to_i != 1)
951-
activityStatuses = 'AND activity_status_code:(2)'
956+
activityStatuses = '&fq=activity_status_code:(2)'
952957
else
953-
activityStatuses = 'AND activity_status_code:(2 OR 3 OR 4 OR 5)'
954-
end
955-
if(!query.start_with?("\""))
956-
tempQ = '"' + query + '"'
957-
isInIdentifier = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=iati_identifier:#{tempQ}&fl=iati_identifier&rows=1")
958-
isInTitle = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=title_narrative_text:#{tempQ}&fl=iati_identifier&rows=1")
959-
checkInID = Oj.load(isInIdentifier)
960-
checkInTitle = Oj.load(isInTitle)
961-
if(checkInID['response']['numFound'].to_i == 1 || checkInTitle['response']['numFound'].to_i == 1)
962-
query = tempQ
963-
else
964-
end
958+
activityStatuses = '&fq=activity_status_code:(2 OR 3 OR 4 OR 5)'
965959
end
960+
# if(!query.start_with?("\""))
961+
# tempQ = '"' + query + '"'
962+
# isInIdentifier = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=iati_identifier:#{tempQ}&fl=iati_identifier&rows=1")
963+
# isInTitle = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=title_narrative_text:#{tempQ}&fl=iati_identifier&rows=1")
964+
# checkInID = Oj.load(isInIdentifier)
965+
# checkInTitle = Oj.load(isInTitle)
966+
# if(checkInID['response']['numFound'].to_i == 1 || checkInTitle['response']['numFound'].to_i == 1)
967+
# query = tempQ
968+
# else
969+
# end
970+
# end
966971
filters = prepareFilters(query.to_s, 'F')
967972
response = solrResponse(query, activityStatuses, 'F', 0, '', '')
968973
if(response['response']['numFound'].to_i > 0)
@@ -1005,14 +1010,80 @@ def getCacheData(fileName)
10051010
}
10061011
end
10071012

1013+
# post '/search_p/?' do
1014+
# #query = params['query']
1015+
# query = sanitize_input(params['query'],"newId")
1016+
# isIncludeClosedProjects = sanitize_input(params['includeClosedProject'],"newId")
1017+
# if(isIncludeClosedProjects.to_i != 1)
1018+
# activityStatuses = 'AND activity_status_code:(2)'
1019+
# else
1020+
# activityStatuses = 'AND activity_status_code:(2 OR 3 OR 4 OR 5)'
1021+
# end
1022+
# if(!query.start_with?("\""))
1023+
# tempQ = '"' + query + '"'
1024+
# isInIdentifier = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=iati_identifier:#{tempQ}&fl=iati_identifier&rows=1")
1025+
# isInTitle = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=title_narrative_text:#{tempQ}&fl=iati_identifier&rows=1")
1026+
# checkInID = Oj.load(isInIdentifier)
1027+
# checkInTitle = Oj.load(isInTitle)
1028+
# if(checkInID['response']['numFound'].to_i == 1 || checkInTitle['response']['numFound'].to_i == 1)
1029+
# query = tempQ
1030+
# else
1031+
# end
1032+
# end
1033+
# filters = prepareFilters(query.to_s, 'F')
1034+
# response = solrResponse(query, activityStatuses, 'F', 0, '', '')
1035+
# if(response['response']['numFound'].to_i > 0)
1036+
# ##Clean any unnecessary activity
1037+
# ## TODO
1038+
# elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
1039+
# if elist.length > 0
1040+
# tempResponseList = response['response']['docs']
1041+
# tempResponseList.each do |data|
1042+
# if elist.include?(data['iati_identifier'])
1043+
# tempResponseList.delete(data)
1044+
# end
1045+
# end
1046+
# response['response']['docs'] = tempResponseList
1047+
# end
1048+
# ##
1049+
# response['response'] = addTotalBudgetWithCurrency(response['response'])
1050+
# response = addHighlightingToFTSTerms(response)
1051+
# end
1052+
# settings.devtracker_page_title = 'Search Results For : ' + query
1053+
# didYouMeanQuery = sanitize_input(params['query'],"a")
1054+
# #didYouMeanData = generate_did_you_mean_data(didYouMeanQuery,'2')
1055+
# #erb :'search/solrSearch',
1056+
# erb :'search/solrTemplate',
1057+
# :layout => :'layouts/layout',
1058+
# :locals =>
1059+
# {
1060+
# oipa_api_url: settings.oipa_api_url,
1061+
# query: query,
1062+
# filters: filters,
1063+
# response: response['response'],
1064+
# solrConfig: Oj.load(File.read('data/solr-config.json')),
1065+
# activityStatus: Oj.load(File.read('data/activity_status.json')),
1066+
# searchType: 'F',
1067+
# breadcrumbURL: '',
1068+
# breadcrumbText: '',
1069+
# fcdoCountryBudgets: nil,#didYouMeanData['dfidCountryBudgets'],
1070+
# fcdoRegionBudgets: nil,#didYouMeanData['dfidRegionBudgets'],
1071+
# isIncludeClosedProjects: isIncludeClosedProjects
1072+
# }
1073+
# end
1074+
10081075
post '/solr-response' do
10091076
query = sanitize_input(params['data']['query'],"newId")
1077+
searchType = sanitize_input(params['data']['queryType'],"newId")
10101078
if params['data']['filters'].strip.length > 1
1011-
filters = 'AND ' + sanitize_input(params['data']['filters'],"newId")
1079+
if searchType == 'F'
1080+
filters = '&fq=' + sanitize_input(params['data']['filters'],"newId").gsub(/\band\b/, " &fq= ")
1081+
else
1082+
filters = 'AND ' + sanitize_input(params['data']['filters'],"newId")
1083+
end
10121084
else
10131085
filters = ''
10141086
end
1015-
searchType = sanitize_input(params['data']['queryType'],"newId")
10161087
startPage = sanitize_input(params['data']['page'],"newId")
10171088
response = solrResponse(query, filters, searchType, startPage, sanitize_input(params['data']['dateRange'],"newId"), sanitize_input(params['data']['sortType'],"newId"))
10181089
if searchType == 'F'

‎helpers/project_helpers.rb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,12 @@ def check_if_project_exists(projectId)
3232
break
3333
end
3434
end
35-
elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
36-
if elist.length > 0
37-
if elist.include?(projectId.to_s)
38-
isGovOrgPresent = false
39-
end
40-
end
35+
# elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
36+
# if elist.length > 0
37+
# if elist.include?(projectId.to_s)
38+
# isGovOrgPresent = false
39+
# end
40+
# end
4141
if !isGovOrgPresent
4242
halt 404, "Activity not found"
4343
end

0 commit comments

Comments
 (0)