Querying an HTTP Source

This topic provides details about the structure to use when writing GDI queries to read or ingest data from HTTP data sources. It also includes example queries that may be useful as a starting point for writing your own GDI queries.

• Query Syntax
• Mapping the Content Property to JSON
• Query Examples

Query Syntax

The following query syntax shows the structure of a GDI query for HTTP sources. The clauses, patterns, and placeholders that are links are described below.

# PREFIX Clause
PREFIX s:     <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX rdf:   <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs:  <http://www.w3.org/2000/01/rdf-schema#>
PREFIX xsd:   <http://www.w3.org/2001/XMLSchema#>
PREFIX owl:   <http://www.w3.org/2002/07/owl#>
PREFIX anzo:  <http://openanzo.org/ontologies/2008/07/Anzo#>
PREFIX zowl:  <http://openanzo.org/ontologies/2009/05/AnzoOwl#>
PREFIX dc:    <http://purl.org/dc/elements/1.1/>

# Result Clause
{ 
   [ GRAPH ${targetGraph} { ] 
   triple_patterns
 [ } ]
}
[ ${usingSources} ]

WHERE
{
   # SERVICE Clause: Include the following service call when reading or inserting data.
    SERVICE [ TOPDOWN ] <http://cambridgesemantics.com/services/DataToolkit>

   # View SERVICE Clause: Or use the service call below when constructing a view.
    SERVICE <http://cambridgesemantics.com/services/DataToolkitView>(${targetGraph})

    { 
      ?data a s:HttpSource ;
        s:url "string" ;
        [ s:authorization [ 
            a s:BearerToken ; s:token "string" ;
          | a s:AWSSignature ; s:accessKey "string" ; s:region "string" ;
                   s:secretKey "string" ; s:serviceName "string" ;
                   s:sessionToken "string" ;
          | a s:BasicAuth ; s:username "string" ; s:password "string" ;
        ] ; ]
        [ s:trust "string" ; ]
        [ s:proxy "string" | [ s:host "string" ; s:port int ] ]
        [ s:header [ s:name: "string" ; s:value "string" ] ; ]
        [ s:mimetype "string" ; ]
        [ s:contentType "string" ; ]
        [ s:content """string""" ; ]
        [ s:parameter [ s:name "string" ; s:value "string" ] ; ]
        [ s:method "string" ; ]
        [ s:encoding "string" ; ]
        [ s:form [ s:name: "string" ; s:value "string" ] ; ]
        [ s:format [ source_format_options ; ] ; ]
        [ s:timeout int ; ]
        [ s:batching boolean | int ; ]
        [ s:paging [ pagination_options ; ]
        [ s:concurrency int | [ list_of_properties ] ; ]
        [ s:rate int | "string" ; ]
        [ s:partitionBy "string" | ?variable ; ]
        [ s:locale "string" ; ]
        [ s:sampling int ; ]
        [ s:selector "string" | [ list ] ; ]
        [ s:model "string" ; ]
        [ s:key ("string") ; ]
        [ s:reference [ s:model "string" ; s:using ("string") ]
        [ s:formats [ datatype_formatting_options ] ; ]
        [ s:normalize boolean | [ normalization_rules ] ; ]
        [ s:count ?variable ; ]
        [ s:offset int ; ]
        [ s:orderBy "string" | ?variable ; ]
        [ s:limit int ; ]
        # Mapping variables
        ?mapping_variable ( [ "binding" ] [ datatype ] [ "datetime_format" ] ) ;
        ... ;
        .
     # Additional clauses such as BIND, VALUES, FILTER
   }
}

PREFIX s:      <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX rdf:    <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs:   <http://www.w3.org/2000/01/rdf-schema#>
PREFIX xsd:    <http://www.w3.org/2001/XMLSchema#>
PREFIX owl:    <http://www.w3.org/2002/07/owl#>
PREFIX anzo:   <http://openanzo.org/ontologies/2008/07/Anzo#>
PREFIX zowl:   <http://openanzo.org/ontologies/2009/05/AnzoOwl#>
PREFIX dc:     <http://purl.org/dc/elements/1.1/>

?data a s:HttpSource ;
  s:url "{{@db.eca4bfa83481f3638b93ab5fdf93ff9a.url}}" ;
  s:authorization [ a s:BasicAuth ; 
    s:username "{{@db.eca4bfa83481f3638b93ab5fdf93ff9a.user}}" ;
    s:password "{{@db.eca4bfa83481f3638b93ab5fdf93ff9a.password}}" ;
  ] ;

s:authorization [ a s:BearerToken | s:AWSSignature | s:BasicAuth ] ;

s:authorization [ a s:BearerToken ;
   s:token "string"
] ;

s:authorization [ 
   a s:AWSSignature ; s:accessKey "string" ;
   s:region "string" ; s:secretKey "string" ;
   s:serviceName "string" ; s:sessionToken "string" ;
] ;

  s:authorization [ a s:BasicAuth ; 
    s:username "string" ;
    s:password "string" ;
  ] ;

 s:header [ s:name "Accept" ; s:value "application/json" ]

s:parameter [ s:name "format" ; s:value "csv" ] ,
            [ s:name "named-graph-uri" ; s:value "http://cambridgesemantics.com/Layer/d541..." ] ;

SERVICE <http://cambridgesemantics.com/services/DataToolkit> {
  VALUES (?reviews_text) {
  ("Horrible, terrible, will never use again.")
  ("Wonderful, magnificent, will recommend to everyone!")
  ("Simply OK. I might purchase this product again.") }

  ?data a s:HttpSource ;
    s:url "https://api.meaningcloud.com/sentiment-2.1" ;
    s:contentType "multipart/form-data" ;
    s:form [ s:name "key" ; s:value "9eab751142..." ],
           [ s:name "lang" ; s:value "auto" ],
           [ s:name "txt" ; s:value ?reviews_text ] ;
    ?confidence ();
    ?score_tag ();
    ?subjectivity () ;
    ?irony () ;
    ?agreement () .
}

s:concurrency [
  s:limit 24 ;
  s:nodes 4 ;
  s:executorsPerNode 8 ;
] ;

s:rate "90/minute" ;
s:rate "90 per minute" ;
s:rate "200000 every week" ;
s:rate "10000 every 6 hours" ;

Mapping the Content Property to JSON

The s:content property can be configured with an inline object (blank node) that gets translated to JSON in the request body. This mapping allows for creation of embedded objects and arrays as well as a mechanism for iterating over all available input so that HTTP endpoints that support batching can be used more effectively.

Using Blank Nodes

Blank nodes are used to create an object in the output JSON. The local name of any predicate used within content becomes a key in the generated JSON object. Blank nodes can be embedded within each other, allowing the hierarchical nature of JSON to be represented. For example:

s:content [ ex:firstName "Mary" ; ex:lastName "Barry" ] ;

Or

s:content [ ex:person [ ex:firstName "Mary" ] ] ;

Using Variables

Variables can be also used in the object position to construct a request from input at runtime. For example:

s:content [ ex:firstName ?firstName ; ex:lastName ?lastName ] ;

The values for the variables can come from a TOPDOWN variable, a VALUES clause in the SERVICE block, or another data source. Any unbound variables in the input will not be added to the generated JSON object.

Using RDF Lists

An RDF list can also be used to create an array in the output JSON. For example:

s:content [ ex:allKnownNames ( ?firstName ?lastName ?nickName ) ]

An RDF list can also be embedded inside another list to create an array in the output JSON and populate it with items evaluated against a repeating pattern across all available input rows for a slice. That pattern can be a variable, which generates an array of primitive values, or a blank node, which generates an array of mapped JSON objects. For example:

s:content [ ex:documents ((?id)) ] ;

Or

s:content [ ex:documents (([ ex:id ?id ; ex:title ?title ])) ;

Example

The following example query demonstrates the use of s:content to generate JSON. The query also includes the s:concurrency property to restrict execution to a single slice. Without limiting execution when there are a small number of inputs (as in the VALUES clause), each input row gets executed on its own. As the inputs increase, each slice operates over a larger number of inputs until the default s:batching 5000 is applied.

PREFIX s: <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
PREFIX api: <http://contoso.com/api/>

SELECT	*
WHERE {
    SERVICE TOPDOWN <http://cambridgesemantics.com/services/DataToolkit>
   {
      VALUES (?firstName ?lastName ?dob ?email)
      {
        ("Gray" "Hay" "1978-03-18"^^xsd:date "gray@abc.com")
        ("Ana" "Bana" "1974-10-20"^^xsd:date "ana@abc.com")
        ("George" "Forge" "1975-08-13"^^xsd:date "george@abc.com")
        ("Miles" "Giles" "1977-04-12"^^xsd:date "miles@abc.com")
      }

      ?data a s:HttpSource ;
        s:url "https://postman-echo.com/post" ;
        s:header [ s:name "Accept" ; s:value "application/json" ] ;
        s:concurrency 1 ;
        s:content
        (([
           api:dateOfBirth ?dob ;
           api:email ?email ;
           api:year 2020 ;
           api:person [ api:firstName ?firstName ; api:lastName ?lastName ] ;
        ])) ;
        s:selector "data" ;
        ?firstName ("person.firstName" xsd:string) ;
        ?lastName ("person.lastName" xsd:string) ;
        ?dob ("dateOfBirth" xsd:date) ;
        ?email ("email" xsd:string) ;
        ?year ("year" xsd:int) .
  }
}

The content portion of the request that the query generates is shown below:

[{
   "firstName": "Gray" ,
   "lastName": "Hay" ,
   "dateOfBirth": "1978-03-18" ,
   "email": "gray@abc.com" ,
   "year": 2020
  },
  {
   "firstName": "Ana" ,
   "lastName": "Bana" ,
   "dateOfBirth": "1974-10-20" ,
   "email": "ana@abc.com" ,
   "year": 2020
  },
  {
    ...
}]

Query Examples

• Topdown Query with URL Parameters
• Generator Query against an Anzo SPARQL Endpoint
• API Queries

Topdown Query with URL Parameters

The query below reads data from a sample HTTP source that compiles worldwide weather statistics. The source has several models available for retrieving data that is current, daily, historical, etc. To target current data, the query includes s:selector "currently". In addition, the query demonstrates the use of the "topdown" functionality, where the query sends values to the source to narrow the results. The VALUES clause specifies the latitude and longitude values for the cities to return data for. In addition, since this sample source requires parameters to be specified in the connection URL, the s:url value includes ?lat and ?long as parameters as part of the value.

PREFIX s: <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
PREFIX ex: <http://example.org/ontologies/City#>

SELECT  
	?city ?state ?temp ?rainChance
	?humidity ?pressure ?windSpeed
WHERE
{
	SERVICE TOPDOWN <http://cambridgesemantics.com/services/DataToolkit>
    {
	  ?data a s:HttpSource ;
	  s:url "https://sampleEndpoint.com/forecast/{{?lat}},{{?long}}" ;
	  s:selector "currently" ;
	  ?lat ("latitude") ;
	  ?long ("longitude") ;
	  ?temp ("temperature") ;
	  ?rainChance ( "precipProbability" ) ;
	  ?humidity () ;
	  ?pressure () ;
	  ?windSpeed () .
    }
    VALUES( ?city ?state ?lat ?long )
    {
	  ( "Lakeway" "TX" 30.374563 -97.975892 )
	  ( "Boston" "MA" 42.358043 -71.060415 )
	  ( "Seattle" "WA" 47.590720 -122.307053 )
	  ( "Chicago" "IL" 41.837741 -87.823296 )
	  ( "Hilo" "HI" 19.702040 -155.090312 )
    }
}
ORDER BY ?city

Generator Query against an Anzo SPARQL Endpoint

The example below is a GDI Generator query that retrieves graphmart data from a remote Anzo SPARQL endpoint.

PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX s: <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>

INSERT {
  GRAPH ${targetGraph} {
    ?s ?p ?o }
}
${usingSources}
WHERE {
    SERVICE <http://cambridgesemantics.com/services/DataToolkit>
  {
        ?data a s:HttpSource ;
            s:url "https://10.10.0.10/sparql/graphmart/http%3A%2F%2Fcambridgesemantics.com%2FGraphmart%2F1686168b-3eaf-4fdc-9730-1903717b9e62";
            s:trust "all" ;
            s:username "user";
            s:password "pass";
            s:contentType "application/sparql-query" ;
            s:header [ s:name "Accept" ; s:value "text/csv" ] ;
        s:content """
          PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
          SELECT ?s ?p ?o 
          WHERE {
            ?s ?p ?o .
            FILTER(ISLITERAL(?o))
          }
        """ .
        ?rdf a s:RdfGenerator, s:OntologyGenerator ;
            s:as (?s ?p ?o) ;
            s:ontology <http://cambridgesemantics.com/ontologies/TopMovies> ;
            s:base <http://cambridgesemantics.com/data> .
  }
}

API Queries

The following example queries the Google Recognize API to request transcriptions for voice recordings that are stored in a Google bucket.

PREFIX s: <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>
PREFIX anzo: <http://openanzo.org/ontologies/2008/07/Anzo#>
PREFIX zowl: <http://openanzo.org/ontologies/2009/05/AnzoOwl#>
PREFIX dc: <http://purl.org/dc/elements/1.1/>

INSERT {
GRAPH ${targetGraph}{
    ?record <http://google.com/transcript> ?transcript .
    ?record <http://google.com/confidence> ?confidence .
    ?record <http://google.com/file> ?file .
  }
}
${usingSources}
WHERE {
    BIND(<gs://csi-se/demo/emergency-test.mp3> as ?file)
    BIND(UUID() as ?record)
{
        SERVICE <http://cambridgesemantics.com/services/DataToolkit>
        {
            ?data a s:HttpSource ;
              s:selector "results.alternatives" ;
              s:url "https://speech.googleapis.com/v1p1beta1/speech:recognize" ;
              s:authorization [ a s:BearerToken ; s:token """ya29...""" ] ;
              s:content """
    {
      "config": {
        "encoding":"MP3",
        "sampleRateHertz": 16000,
        "languageCode": "en-US",
        "enableWordTimeOffsets": false
    },
      "audio": {
        "uri":"gs://csi-se/demo/emergency-test.mp3"}
    }
    """ ;
            ?confidence ("confidence") ;
            ?transcript ("transcript") .
      } 
   }
}

The example below includes the header and content properties to send a request that contains small text snippets for sentiment analysis.

PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX s: <http://cambridgesemantics.com/ontologies/DataToolkit#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
PREFIX ont: <http://cambridgesemantics.com/ontologies/Sentiment_Analysis#>

INSERT {
  GRAPH ${targetGraph} {
    ?requirement a ont:Sentiment ;
      ont:p_Sentiment_Type ?sentiment ;
      ont:p_Sentiment_Score ?polarity .
  }
}
${usingSources}
WHERE {
  ?requirement a <http://cambridgesemantics.com/Layer/3b4163e7f53149d5a815627be5d409bd/Model#Requirements> ;
  <http://cambridgesemantics.com/Layer/3b4163e7f53149d5a815627be5d409bd/Model#Requirements.reqText> ?requirement_text .

  SERVICE TOPDOWN <http://cambridgesemantics.com/services/DataToolkit> {
      ?data a s:HttpSource ;
        s:url "https://text-analysis12.p.rapidapi.com/sentiment-analysis/api/v1.1" ;
        s:method "POST" ;
        s:header [ s:name "Accept" ; s:value "application/json" ] ,
                 [ s:name "X-RapidAPI-Key" ; s:value "key" ] ,
                 [ s:name "X-RapidAPI-Host" ; s:value "text-analysis12.p.rapidapi.com" ] ;
        s:contentType "application/json" ;
        s:content """{ "text": "{{?requirement_text}}" , "language": "english" }""" ;
        ?polarity ("aggregate_sentiment/compound" xsd:double);
        ?sentiment () .
  }
}