I am trying to import data from files like CSV, XML and Json to solr core, I am new to solr so this might be a straightforward to some but for me I have tried many online suggestions but not getting the desired result.
So I have a json file and I have enabled dataimport by adding the following requestHandler to solrconfig.xml:
<requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
<lst name="defaults">
<str name="config">solr-data-config.xml</str>
</lst>
</requestHandler>
And in the solr-data-config.xml:
<dataConfig>
<dataSource name="dfs" type="FileDataSource"/>
<document>
<entity name="sourcefile" processor="FileListEntityProcessor" fileName=".*" rootEntity="false" baseDir="${solr.install.dir}/example/exampledocs">
<entity name="entryline" processor="LineEntityProcessor" url="${sourcefile.fileAbsolutePath}" rootEntity="true" dataSource="fds" separator=","/>
</entity>
</document>
The updated version:
<dataConfig>
<script><![CDATA[
function CategoryPieces(row) {
var pieces = row.get('manu_id_s').split('/');
var arr = new Array();
for (var i=0; i < pieces.length; i++) {
row.put('manu_id_s' + i, pieces[i].trim());
arr[i] = pieces[i].trim();
}
row.put('manu_id_s', (pieces.length - 1).toFixed());
row.put('manu_id_s', arr.join('/'));
row.put('manu_id_s', arr.join('/'));
return row;
}
]]></script>
<dataSource type="FileDataSource" />
<document>
<entity
name="document"
processor="FileListEntityProcessor"
baseDir="${solr.install.dir}/example/exampledocs/khaled"
fileName=".*.xml$"
recursive="false"
rootEntity="false"
dataSource="null">
<entity
name="test"
processor="XPathEntityProcessor"
transformer="script:CategoryPieces"
url="${document.fileAbsolutePath}"
useSolrAddSchema="true"
stream="true">
</entity>
</entity>
</document>
</dataConfig>
And I have added a json, csv and xml files to the path, in the ui of solr when I use the dataimport it says that e.g. Requests: 0 , Fetched: 26 , Skipped: 0 , Processed: 0 and nothing in the logs, can someone advice how to add the data in the file to solr?
Here is my sample data in the xml file:
<add>
<doc>
<field name="id">USD</field>
<field name="name">One Dollar</field>
<field name="manu">Bank of America</field>
<field name="manu_id_s">boa</field>
<field name="cat">currency</field>
<field name="features">Coins and notes</field>
<field name="price_c">1,USD</field>
<field name="inStock">true</field>
</doc>
<doc>
<field name="id">EUR</field>
<field name="name">One Euro</field>
<field name="manu">European Union</field>
<field name="manu_id_s">eu</field>
<field name="cat">currency</field>
<field name="features">Coins and notes</field>
<field name="price_c">1,EUR</field>
<field name="inStock">true</field>
</doc>
<doc>
<field name="id">GBP</field>
<field name="name">One British Pound</field>
<field name="manu">U.K.</field>
<field name="manu_id_s">uk</field>
<field name="cat">currency</field>
<field name="features">Coins and notes</field>
<field name="price_c">1,GBP</field>
<field name="inStock">true</field>
</doc>
<doc>
<field name="id">NOK</field>
<field name="name">One Krone</field>
<field name="manu">Bank of Norway</field>
<field name="manu_id_s">nor</field>
<field name="cat">currency</field>
<field name="features">Coins and notes</field>
<field name="price_c">1,NOK</field>
<field name="inStock">true</field>
</doc>
</add>
Now the data returned from the query looks like:
{
"responseHeader":{
"status":0,
"QTime":0,
"params":{
"q":"*:*",
"_":"1547556805546"}},
"response":{"numFound":4,"start":0,"docs":[
{
"manu_id_s":"boa",
"id":"USD",
"_version_":1622731088078045184},
{
"manu_id_s":"eu",
"id":"EUR",
"_version_":1622731088081190912},
{
"manu_id_s":"uk",
"id":"GBP",
"_version_":1622731088081190913},
{
"manu_id_s":"nor",
"id":"NOK",
"_version_":1622731088082239488}]
}}