Skip to content

Commit 7874c44

Browse files
aunindadfaunindadf
committed
Improving search mechanism.
1 parent 55d685f commit 7874c44

3 files changed

Lines changed: 99 additions & 30 deletions

File tree

‎data/solr-config.json‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"APILink": "https://fcdo.iati.cloud/",
2+
"APILink": "https://fcdo2.iati.cloud/",
33
"DefaultFieldsToReturn": "id, iati_identifier, reporting_org_ref, reporting_org_narrative, title_narrative_first, title_narrative, description_narrative, activity_date_iso_date, activity_date_start_actual, activity_date_end_planned, activity_status_code, humanitarian, tag_code, tag, child_aggregation_*, activity_plus_child_aggregation*, activity_aggregation_*, default_currency, activity_date_start_common_f, activity_date_end_common_f, activity_date_start_common, activity_date_end_common",
44
"DefaultFieldsToSearch":
55
[
@@ -13,7 +13,8 @@
1313
"document_link_title_narrative_text",
1414
"transaction",
1515
"reporting_org_narrative",
16-
"participating_org_narrative"
16+
"participating_org_narrative",
17+
"related_activity_context"
1718
],
1819
"SortingOptions":
1920
[
@@ -37,71 +38,71 @@
3738
{
3839
"title": "Activity Status",
3940
"field": "activity_status_code",
40-
"url": "search/activity/?facet=on&facet.field=activity_status_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
41+
"url": "api/v2/activity/?facet=on&facet.field=activity_status_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
4142
"mappingFile": "data/mappingSolrFilters/activity_status.json",
4243
"defaultValue": "2",
4344
"isFieldTypeString": "0"
4445
},
4546
{
4647
"title": "Government Department(s)",
4748
"field": "reporting_org_ref",
48-
"url": "search/activity/?facet=on&facet.field=reporting_org_ref&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
49+
"url": "api/v2/activity/?facet=on&facet.field=reporting_org_ref&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
4950
"mappingFile": "data/custom-codes/ogds.json",
5051
"defaultValue": "",
5152
"isFieldTypeString": "1"
5253
},
5354
{
5455
"title": "Tags",
5556
"field": "tag_code",
56-
"url": "search/activity/?facet=on&facet.field=tag_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
57+
"url": "api/v2/activity/?facet=on&facet.field=tag_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
5758
"mappingFile": "data/custom-codes/policy-priorities.json",
5859
"defaultValue": "",
5960
"isFieldTypeString": "1"
6061
},
6162
{
6263
"title": "Sectors",
6364
"field": "sector_code",
64-
"url": "search/activity/?facet=on&facet.field=sector_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
65+
"url": "api/v2/activity/?facet=on&facet.field=sector_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
6566
"mappingFile": "sectorHierarchies.json",
6667
"defaultValue": "",
6768
"isFieldTypeString": "0"
6869
},
6970
{
7071
"title": "Participating Orgs",
7172
"field": "participating_org_ref",
72-
"url": "search/activity/?facet=on&facet.field=participating_org_ref&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND participating_org_role:4 AND ",
73+
"url": "api/v2/activity/?facet=on&facet.field=participating_org_ref&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND participating_org_role:4 AND ",
7374
"mappingFile": "iati_publishers_list.json",
7475
"defaultValue": "",
7576
"isFieldTypeString": "1"
7677
},
7778
{
7879
"title": "Activity Dates",
7980
"field": "activity_date_iso_date",
80-
"url": "search/activity/?facet=on&facet.field=activity_date_iso_date&facet.limit=-1&facet.mincount=1&q=participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
81+
"url": "api/v2/activity/?facet=on&facet.field=activity_date_iso_date&facet.limit=-1&facet.mincount=1&q=participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
8182
"mappingFile": "",
8283
"defaultValue": "",
8384
"isFieldTypeString": "1"
8485
},
8586
{
8687
"title": "Benefiting Countries",
8788
"field": "recipient_country_code",
88-
"url": "search/activity/?facet=on&facet.field=recipient_country_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
89+
"url": "api/v2/activity/?facet=on&facet.field=recipient_country_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
8990
"mappingFile": "data/country-codes/country_codes.json",
9091
"defaultValue": "",
9192
"isFieldTypeString": "1"
9293
},
9394
{
9495
"title": "Benefiting Regions",
9596
"field": "recipient_region_code",
96-
"url": "search/activity/?facet=on&facet.field=recipient_region_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
97+
"url": "api/v2/activity/?facet=on&facet.field=recipient_region_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
9798
"mappingFile": "data/regionCodes.json",
9899
"defaultValue": "",
99100
"isFieldTypeString": "1"
100101
},
101102
{
102103
"title": "Document Categories",
103104
"field": "document_link_category_code",
104-
"url": "search/activity/?facet=on&facet.field=document_link_category_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
105+
"url": "api/v2/activity/?facet=on&facet.field=document_link_category_code&facet.limit=-1&facet.mincount=1&q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
105106
"mappingFile": "data/custom-codes/DocumentCategory.json",
106107
"defaultValue": "",
107108
"isFieldTypeString": "1"
@@ -111,28 +112,28 @@
111112
{
112113
"F":
113114
{
114-
"url": "/search/activity/?hl=on&hl.requireFieldMatch=true&hl.fl=title_narrative,title_narrative_first,description_narrative,iati_identifier,transaction_description_narrative,reporting_org_ref,recipient_country_name,document_link_title_narrative_text,transaction,reporting_org_narrative,participating_org_narrative&q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND ",
115+
"url": "/api/v2/activity/?q.op=AND&sort=if(termfreq(reporting_org_ref,'GB-GOV-1'),1,if(termfreq(reporting_org_ref,'GB-GOV-2'),2,99)) asc, score desc&hl=on&hl.requireFieldMatch=true&hl.fl=title_narrative,title_narrative_first,description_narrative,iati_identifier,transaction_description_narrative,reporting_org_ref,recipient_country_name,document_link_title_narrative_text,transaction,reporting_org_narrative,participating_org_narrative&fq=hierarchy:1&fq=participating_org_ref:GB-GOV-*",
115116
"fieldDependency": "",
116117
"comments": "This search category is related to the free text search of DevTracker"
117118
},
118119
"C":
119120
{
120-
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
121+
"url": "/api/v2/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
121122
"fieldDependency": "recipient_country_code"
122123
},
123124
"R":
124125
{
125-
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
126+
"url": "/api/v2/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
126127
"fieldDependency": "recipient_region_code"
127128
},
128129
"O":
129130
{
130-
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
131+
"url": "/api/v2/activity/?q=hierarchy:1 AND participating_org_ref:(GB-GOV-* OR GB-COH-*) AND ",
131132
"fieldDependency": "reporting_org_ref"
132133
},
133134
"S":
134135
{
135-
"url": "/search/activity/?q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND ",
136+
"url": "/api/v2/activity/?q=hierarchy:1 AND participating_org_ref:GB-GOV-* AND ",
136137
"fieldDependency": "sector_code"
137138
}
138139
},

‎devtracker.rb‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@
6161
# Developer Machine: set global settings
6262
#set :oipa_api_url, 'https://fcdo-direct-indexing.iati.cloud/search/'#'https://fcdo.iati.cloud/search/'#'https://fcdo-direct-indexing.iati.cloud/search/'#'https://devtracker.fcdo.gov.uk/api/'
6363
# set :oipa_api_url, 'https://devtracker-entry.oipa.nl/api/'
64-
# set :oipa_api_url, 'https://fcdo.iati.cloud/search/'
64+
set :oipa_api_url, 'https://fcdo2.iati.cloud/api/v2/'
6565
# set :oipa_api_url, 'https://fcdo-direct-indexing.iati.cloud/search/'
6666
# set :bind, '0.0.0.0' # Allows for vagrant pass-through whilst debugging
6767

6868
# Server Machine: set global settings to use varnish cache
69-
set :oipa_api_url, 'http://127.0.0.1:6081/search/'
69+
#set :oipa_api_url, 'http://127.0.0.1:6081/search/'
7070
set :prod_api_url, 'https://fcdo.iati.cloud'
7171
set :dev_api_url, 'https://fcdo-staging.iati.cloud'
7272

@@ -948,9 +948,9 @@ def getCacheData(fileName)
948948
query = sanitize_input(params['query'],"newId")
949949
isIncludeClosedProjects = sanitize_input(params['includeClosedProject'],"newId")
950950
if(isIncludeClosedProjects.to_i != 1)
951-
activityStatuses = 'AND activity_status_code:(2)'
951+
activityStatuses = '&fq=activity_status_code:(2)'
952952
else
953-
activityStatuses = 'AND activity_status_code:(2 OR 3 OR 4 OR 5)'
953+
activityStatuses = '&fq=activity_status_code:(2 OR 3 OR 4 OR 5)'
954954
end
955955
if(!query.start_with?("\""))
956956
tempQ = '"' + query + '"'
@@ -1005,6 +1005,68 @@ def getCacheData(fileName)
10051005
}
10061006
end
10071007

1008+
# post '/search_p/?' do
1009+
# #query = params['query']
1010+
# query = sanitize_input(params['query'],"newId")
1011+
# isIncludeClosedProjects = sanitize_input(params['includeClosedProject'],"newId")
1012+
# if(isIncludeClosedProjects.to_i != 1)
1013+
# activityStatuses = 'AND activity_status_code:(2)'
1014+
# else
1015+
# activityStatuses = 'AND activity_status_code:(2 OR 3 OR 4 OR 5)'
1016+
# end
1017+
# if(!query.start_with?("\""))
1018+
# tempQ = '"' + query + '"'
1019+
# isInIdentifier = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=iati_identifier:#{tempQ}&fl=iati_identifier&rows=1")
1020+
# isInTitle = RestClient.get api_simple_log(settings.oipa_api_url + "activity?q=title_narrative_text:#{tempQ}&fl=iati_identifier&rows=1")
1021+
# checkInID = Oj.load(isInIdentifier)
1022+
# checkInTitle = Oj.load(isInTitle)
1023+
# if(checkInID['response']['numFound'].to_i == 1 || checkInTitle['response']['numFound'].to_i == 1)
1024+
# query = tempQ
1025+
# else
1026+
# end
1027+
# end
1028+
# filters = prepareFilters(query.to_s, 'F')
1029+
# response = solrResponse(query, activityStatuses, 'F', 0, '', '')
1030+
# if(response['response']['numFound'].to_i > 0)
1031+
# ##Clean any unnecessary activity
1032+
# ## TODO
1033+
# elist = Oj.load(RestClient.get 'https://iati.fcdo.gov.uk/iati_files/elist.json')
1034+
# if elist.length > 0
1035+
# tempResponseList = response['response']['docs']
1036+
# tempResponseList.each do |data|
1037+
# if elist.include?(data['iati_identifier'])
1038+
# tempResponseList.delete(data)
1039+
# end
1040+
# end
1041+
# response['response']['docs'] = tempResponseList
1042+
# end
1043+
# ##
1044+
# response['response'] = addTotalBudgetWithCurrency(response['response'])
1045+
# response = addHighlightingToFTSTerms(response)
1046+
# end
1047+
# settings.devtracker_page_title = 'Search Results For : ' + query
1048+
# didYouMeanQuery = sanitize_input(params['query'],"a")
1049+
# #didYouMeanData = generate_did_you_mean_data(didYouMeanQuery,'2')
1050+
# #erb :'search/solrSearch',
1051+
# erb :'search/solrTemplate',
1052+
# :layout => :'layouts/layout',
1053+
# :locals =>
1054+
# {
1055+
# oipa_api_url: settings.oipa_api_url,
1056+
# query: query,
1057+
# filters: filters,
1058+
# response: response['response'],
1059+
# solrConfig: Oj.load(File.read('data/solr-config.json')),
1060+
# activityStatus: Oj.load(File.read('data/activity_status.json')),
1061+
# searchType: 'F',
1062+
# breadcrumbURL: '',
1063+
# breadcrumbText: '',
1064+
# fcdoCountryBudgets: nil,#didYouMeanData['dfidCountryBudgets'],
1065+
# fcdoRegionBudgets: nil,#didYouMeanData['dfidRegionBudgets'],
1066+
# isIncludeClosedProjects: isIncludeClosedProjects
1067+
# }
1068+
# end
1069+
10081070
post '/solr-response' do
10091071
query = sanitize_input(params['data']['query'],"newId")
10101072
if params['data']['filters'].strip.length > 1

‎helpers/solr_helper.rb‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,11 @@ def solrResponse(query, filters, queryType, startPage, dateRange, sortType)
228228
if(sortType != '')
229229
mainQueryString = mainQueryString + '&sort=' + sortType
230230
end
231+
mainQueryString = mainQueryString + '&defType=edismax&qf=' + solrConfig['DefaultFieldsToSearch'].join(' ') + '&mm=20&q=' + query
231232
# Get response based on the API responses
233+
# Embed the following
234+
# https://fcdo2.iati.cloud//api/v2/activity/?q.op=AND&sort=if(termfreq(reporting_org_ref,%27GB-GOV-1%27),1,if(termfreq(reporting_org_ref,%27GB-GOV-2%27),2,99))%20asc,%20score%20desc&fq=hierarchy:1&fq=participating_org_ref:GB-GOV-*&fq=activity_status_code:(2)&defType=edismax&qf=title_narrative%20title_narrative_first%20description_narrative%20iati_identifier%20transaction_description_narrative%20reporting_org_ref%20recipient_country_name%20document_link_title_narrative_text%20transaction%20reporting_org_narrative%20participating_org_narrative%20related_activity_context&mm=20&q=programme%20sudan&rows=20&fl=iati_identifier,title,description*&start=0
235+
#puts apiLink + queryCategory['url'] + mainQueryString +'&rows='+solrConfig['PageSize'].to_s+'&fl='+solrConfig['DefaultFieldsToReturn']+'&start='+startPage.to_s
232236
response = Oj.load(RestClient.get api_simple_log(apiLink + queryCategory['url'] + mainQueryString +'&rows='+solrConfig['PageSize'].to_s+'&fl='+solrConfig['DefaultFieldsToReturn']+'&start='+startPage.to_s))
233237
response
234238
elsif(queryType == 'R')
@@ -278,16 +282,17 @@ def prepareQuery(query, queryType)
278282
mainQueryString = ' ('
279283
if(queryType == 'F')
280284
queryCategory = solrConfig['QueryCategories'][queryType]
281-
if(queryCategory['fieldDependency'] == '')
282-
solrConfig['DefaultFieldsToSearch'].each_with_index do |fieldToBeSearched, index|
283-
if (solrConfig['DefaultFieldsToSearch'].length - 1 == index)
284-
mainQueryString = mainQueryString + fieldToBeSearched + ':' + query.to_s + ''
285-
else
286-
mainQueryString = mainQueryString + fieldToBeSearched + ':' + query.to_s + ' OR '
287-
end
288-
end
289-
mainQueryString = mainQueryString + ')'
290-
end
285+
# if(queryCategory['fieldDependency'] == '')
286+
# solrConfig['DefaultFieldsToSearch'].each_with_index do |fieldToBeSearched, index|
287+
# if (solrConfig['DefaultFieldsToSearch'].length - 1 == index)
288+
# mainQueryString = mainQueryString + fieldToBeSearched + ':' + query.to_s + ''
289+
# else
290+
# mainQueryString = mainQueryString + fieldToBeSearched + ':' + query.to_s + ' OR '
291+
# end
292+
# end
293+
# mainQueryString = mainQueryString + ')'
294+
# end
295+
mainQueryString = ''
291296
elsif(queryType == 'R')
292297
queryCategory = solrConfig['QueryCategories'][queryType]
293298
if(queryCategory['fieldDependency'] != '')
@@ -336,6 +341,7 @@ def prepareQuery(query, queryType)
336341
end
337342
end
338343
end
344+
puts mainQueryString
339345
mainQueryString
340346
end
341347

0 commit comments

Comments
 (0)