Skip to content

Working with vector contexts

There are various methods of a accessing vector context data (as well known as vector elements data).

In NXCALS each context uses CMW entity with a device = "nxcals_context" and property name which corresponds to context name.

For example we have the following device/property pairs defined in a form of an entity:

nxcals_context/BOFSU_COD_H
nxcals_context/BOFSU_COD_V
nxcals_context/LHC.BLMI
nxcals_context/LHC.BLMI.FAST
...

Each entity contains set of (time stamped) rows holding context data for a given period of time. Data timestamp indicates beginning of the context validity range which ends up with timestamp of the following record.

In order to see all the context defined in NXCALS one can use the query below starting with:

Importing necessary libraries including "functions" module for aggregates

import pandas as pd
from nxcals.api.extraction.data.builders import DataQuery
from pyspark.sql.functions import min, max, count, col
import cern.nxcals.api.domain.SystemSpec;
import cern.nxcals.api.domain.Variable;
import cern.nxcals.api.extraction.data.builders.DataQuery;
import cern.nxcals.api.extraction.metadata.InternalServiceClientFactory;
import cern.nxcals.api.extraction.metadata.VariableService;
import cern.nxcals.api.extraction.metadata.queries.Variables;
import cern.nxcals.internal.extraction.metadata.InternalSystemSpecService;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.apache.spark.sql.functions.col;
import static org.apache.spark.sql.functions.count;
import static org.apache.spark.sql.functions.max;
import static org.apache.spark.sql.functions.min;

Retrieving data

df = DataQuery.builder(spark).byEntities().system('CMW') \
    .startTime('2009-01-01 00:00:00.000').endTime('2020-12-31 23:59:59.999') \
    .entity().keyValuesLike({'device': 'nxcals_context', 'property': '%'}) \
    .build().sort(col("property")).select('nxcals_entity_id', 'property',
                                          col('__record_timestamp__').alias("tstamp"))

pdf=df.groupBy('property', 'nxcals_entity_id')\
.agg(min('tstamp').alias('min_t'), max('tstamp').alias('max_t'), count('tstamp').alias('nr'))\
.toPandas()

pdf['min_t'] = pd.to_datetime(pdf.min_t, unit='ns')
pdf['max_t'] = pd.to_datetime(pdf.max_t, unit='ns')

pd.options.display.width = 120
print(pdf)
Map<String, Object> keyValuesLike = new HashMap<String, Object>();
keyValuesLike.put("device", "nxcals_context");
keyValuesLike.put("property", "%");

Dataset<Row> df = DataQuery.builder(spark).byEntities().system("CMW")
        .startTime("2009-01-01 00:00:00.000").endTime("2020-12-31 23:59:59.999")
        .entity().keyValuesLike(keyValuesLike)
        .build().sort(col("property"))
        .select(col("nxcals_entity_id"), col("property"), col("__record_timestamp__").alias("tstamp"));

Dataset<Row> c_df=df.groupBy("property", "nxcals_entity_id")
.agg(min("tstamp").alias("min_t"), max("tstamp").alias("max_t"), count("tstamp").alias("nr"));

c_df.show();
Click to see expected application output...
             property  nxcals_entity_id               min_t               max_t  nr
0         BOFSU_BPM_H           1761456 2011-01-01 00:00:00 2015-04-01 00:00:00   3
1         BOFSU_BPM_V           1761467 2011-01-01 00:00:00 2015-04-01 00:00:00   3
2         BOFSU_COD_H           1761455 2011-01-01 00:00:00 2015-04-01 00:00:00   2
3         BOFSU_COD_V           1761462 2011-01-01 00:00:00 2015-04-01 00:00:00   2
4            LHC.BLMI           1761466 2011-08-19 00:00:00 2018-04-06 22:00:01  25
5       LHC.BLMI.FAST           1761457 2015-03-03 12:00:00 2017-04-05 16:00:00   7
6            LHC.BLMS           1761464 2011-08-19 00:00:00 2018-02-01 09:00:00  13
7       LHC.BLMS.FAST           1761463 2015-03-03 12:00:00 2017-04-05 16:00:00   6
8  ZS.BA2.SPARK_RATES           1761465 2011-07-19 15:43:33 2011-07-19 15:43:33   1

Note

  1. min_t, max_t indicates range in which fall start timestamps of a given context.
  2. The limit defined in the system for the oldest possible context is set to 2009-01-01 00:00:00.000

One of the possible ways to see all the values for a given context is to use DataQuery builder for entities as in the example below:

e_df = DataQuery.builder(spark).byEntities() \
    .system('CMW') \
    .startTime('2009-01-01 00:00:00.000').endTime('2020-12-31 23:59:59.999') \
    .entity() \
    .keyValues({'device': 'nxcals_context', 'property': 'LHC.BLMI'}) \
    .build().select('__record_timestamp__', 'context_value').toPandas()

e_df['__record_timestamp__'] = pd.to_datetime(e_df.__record_timestamp__, unit='ns')

pd.options.display.width = 120
print(e_df)
Map<String, Object> keyValues = new HashMap<String, Object>();
keyValues.put("device", "nxcals_context");
keyValues.put("property", "LHC.BLMI");

Dataset<Row> e_df = DataQuery.builder(spark).byEntities()
        .system("CMW")
        .startTime("2009-01-01 00:00:00.000").endTime("2020-12-31 23:59:59.999")
        .entity()
        .keyValues(keyValues)
        .build().select("__record_timestamp__", "context_value");

 e_df.show();
Click to see expected application output...
            __record_timestamp__                                      context_value
0  2018-04-06 22:00:01.000000000  ([BLMEL.01R1.B2I10_BPMSW.1R1, BLMQI.01R1.B2I30...
1  2017-01-01 00:00:00.000000000  ([BLMEL.01R1.B2I10_BPMSW.1R1, BLMQI.01R1.B2I30...
2  2016-09-16 17:16:30.000000000  ([BLMEL.01R1.B2I10_BPMSW.1R1, BLMQI.01R1.B2I30...
3  2016-09-16 00:00:01.000000000  ([BLMEL.01R1.B2I10_BPMSW.1R1, BLMQI.01R1.B2I30...
4  2016-09-16 00:00:00.000000000  ([BLMEL.01R1.B2I10_BPMSW.1R1, BLMQI.01R1.B2I30...
5  2016-04-28 12:57:37.000000000  ([BLMEL.01R1.B2I10_BPMSW.1R1, BLMQI.01R1.B2I30...
6  2015-08-31 07:00:00.000000000  ([BLMQI.01R1.B2I30_MQXA, BLMEL.01R1.B2I10_BPMS...
7  2015-06-19 14:00:00.000000000  ([BLMQI.01R1.B2I30_MQXA, BLMEL.01R1.B2I10_BPMS...
8  2015-04-15 08:13:12.000759404  ([BLMQI.01R1.B2I30_MQXA, BLMEL.01R1.B2I10_BPMS...
9  2015-02-02 09:00:00.000000000  ([BLMQI.01R1.B2I30_MQXA, BLMEL.01R1.B2I10_BPMS...
10 2013-01-24 07:37:01.000388000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
11 2013-01-22 23:25:41.000976000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
12 2012-11-09 10:08:05.000264000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
13 2012-11-06 20:28:48.000916000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
14 2012-04-24 16:13:04.000782000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
15 2012-03-07 12:22:42.000080000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
16 2012-02-22 10:11:43.000456000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
17 2012-02-20 09:00:51.000042000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
18 2011-11-29 17:40:57.000922000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
19 2011-11-29 17:05:58.000030000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
20 2011-11-25 12:32:29.000072000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
21 2011-09-05 09:28:45.000279000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
22 2011-09-02 07:48:38.000175000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
23 2011-09-01 14:33:16.000589000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...
24 2011-08-19 00:00:00.000000000  ([BLMQI.01R1.B2I30_MQXA, BLMQI.01R1.B1E10_MQXA...

Associating vectornumeric variables with context data

Vector numeric variables can be associated to context data through dedicated variables having their name in the format: NXCALS_CONTEXT:XXXXX where XXXXX represents the vector numeric variable id. NXCALS_CONTEXT:XXXXX variables point to context data itself.

In this model many vector numeric variables can be associated with one context data.

Example:

There are 2 vector numeric variables: LHC.BLMI:CABLE_CONNECTED and LHC.BLMI:LOSS_RS01. Their ids can be verified using the code snippet below:

# NXCALS Metadata API is not yet available for Python
final String CMW_SYSTEM_NAME = "CMW";
final List<String> variableNames =  Arrays.asList("LHC.BLMI:CABLE_CONNECTED", "LHC.BLMI:LOSS_RS01");
InternalSystemSpecService systemService = InternalServiceClientFactory.createSystemSpecService();
SystemSpec systemData = systemService.findByName(CMW_SYSTEM_NAME)
        .orElseThrow(() -> new IllegalArgumentException("No such system name " + CMW_SYSTEM_NAME));

VariableService variableService = InternalServiceClientFactory.createVariableService();

Set<Variable> foundVariables = variableService
        .findAll(Variables.suchThat().systemName().eq(systemData.getName()).and().variableName().in(variableNames));

foundVariables.forEach(v -> System.out.println(v.getVariableName() + " : " + v.getId()));
Click to see expected application output...
LHC.BLMI:LOSS_RS01 : 85147928
LHC.BLMI:CABLE_CONNECTED : 85147984

The variables above are associated with the context data through NXCALS_CONTEXT:85147928 and NXCALS_CONTEXT:85147984. Both of the "association variables" are pointing to LHC.BLMI context data.

The example below selects 1024th element from different contexts defined between years 2015 and 2018:

v_df = DataQuery.builder(spark).byVariables().system("CMW").startTime("2015-04-29 00:00:00.000").endTime("2018-04-30 00:00:00.000") \
    .variable("NXCALS_CONTEXT:85147928").build().withColumn("elements", col("nxcals_value")["elements"]) \
    .select('nxcals_timestamp', "elements")

for row in v_df.collect():
    print(str(row.nxcals_timestamp) + " " + row.elements[1023])
Dataset<Row> v_df = DataQuery.builder(spark).byVariables()
        .system("CMW")
        .startTime("2015-04-29 00:00:00.000").endTime("2018-04-30 00:00:00.000")
        .variable("NXCALS_CONTEXT:85147928")
        .build().withColumn("elements", col("nxcals_value").getField("elements")).select("nxcals_timestamp", "elements");

int ts_idx = v_df.schema().fieldIndex("nxcals_timestamp");
int el_idx = v_df.schema().fieldIndex("elements");

v_df.collectAsList().forEach(r -> {
    System.out.println(r.get(ts_idx) + " " + r.getList(el_idx).get(1023));
});
Click to see expected application output...
1523052001000000000 BLMEI.06L3.B1I10_TCHSH.6L3.B1
1483228800000000000 BLMEI.05L3.B2E10_TCSM.A5L3.B2
1474046190000000000 BLMEI.05L3.B2E10_TCSM.A5L3.B2
1473984001000000000 BLMEI.05L3.B2E10_TCSM.A5L3.B2
1473984000000000000 BLMTI.05L3.B2E10_TCLA.B5L3.B2
1461848257000000000 BLMTI.05L3.B2E10_TCSG.A5L3.B2
1441004400000000000 BLMEI.04L3.B2E10_TCSM.4L3.B2
1434722400000000000 BLMEI.04L3.B2E10_TCSM.4L3.B2

Backport API

An equivalent of the methods available in the old CALS logging service for retrieving of the vector contexts data is provided via Java Backport API. There are 2 methods accessible through MetaDataService, having the following signatures:

VectornumericElementsSet getVectorElements(Variable variable);
VectornumericElementsSet getVectorElementsInTimeWindow(Variable variable, Timestamp minStamp, Timestamp maxStamp);
As an input both methods take vector numeric variable to which context data is attached. Examples of their usage can be found in the code snippets below:

private static void getVectorElements() {
        final String variableName = "LHC.BLMI:LOSS_RS01";

        StringBuilder outputMsg = new StringBuilder();

        MetaDataService metaDataService = ServiceBuilder.getInstance().createMetaService();
        Variable variable = getVariable(variableName);
        VectornumericElementsSet vectornumericElementsSet = metaDataService.getVectorElements(variable);

        outputMsg.append("Listing number of elements for all the contexts attached to " + variableName + ":\n");
        vectornumericElementsSet.getVectornumericElements().forEach((key, value) -> {
            outputMsg.append(key).append(": ").append(value.size()).append("\n");
        });

        System.out.println(outputMsg.toString());
    }
Click to see expected application output...
Listing number of elements for all the contexts attached to LHC.BLMI:LOSS_RS01:
2011-08-19 02:00:00.0: 3607
2011-09-01 16:33:16.000589: 3610
2011-09-02 09:48:38.000175: 3610
2011-09-05 11:28:45.000279: 3610
2011-11-25 13:32:29.000072: 3613
2011-11-29 18:05:58.00003: 3610
2011-11-29 18:40:57.000922: 3613
2012-02-20 10:00:51.000042: 3617
2012-02-22 11:11:43.000456: 3614
2012-03-07 13:22:42.00008: 3621
2012-04-24 18:13:04.000782: 3627
2012-11-06 21:28:48.000916: 3628
2012-11-09 11:08:05.000264: 3629
2013-01-23 00:25:41.000976: 3636
2013-01-24 08:37:01.000388: 3636
2015-02-02 10:00:00.0: 3737
2015-04-15 10:13:12.000759404: 3738
2015-06-19 16:00:00.0: 3738
2015-08-31 09:00:00.0: 3738
2016-04-28 14:57:37.0: 3741
2016-09-16 02:00:00.0: 3747
2016-09-16 02:00:01.0: 3743
2016-09-16 19:16:30.0: 3743
2017-01-01 01:00:00.0: 3746
2018-04-07 00:00:01.0: 3761
private static void getVectorElementsInTimeWindow() {
        final String variableName = "LHC.BLMI:LOSS_RS02";

        StringBuilder outputMsg = new StringBuilder();

        MetaDataService metaDataService = ServiceBuilder.getInstance().createMetaService();
        Variable variable = getVariable(variableName);
        VectornumericElementsSet vectornumericElementsSet = metaDataService.getVectorElementsInTimeWindow(variable,
                TimestampFactory.parseUTCTimestamp("2016-01-01 00:00:00.000"),
                TimestampFactory.parseUTCTimestamp("2017-01-01 00:00:00.000"));

        outputMsg.append("Listing first 10 elements for contexts attached to " + variableName + " in 2016:\n");
        vectornumericElementsSet.getVectornumericElements().forEach((key, value) -> {
            outputMsg.append(key).append(": ");

            StringJoiner elements = new StringJoiner(",");
            for (int idx = 0; idx < value.size() && idx < 10; idx++) {
                elements.add(value.getElementName(idx));
            }
            outputMsg.append(elements).append("\n");
        });

        System.out.println(outputMsg.toString());
    }
Click to see expected application output...
Listing first 10 elements for contexts attached to LHC.BLMI:LOSS_RS02 in 2016:
2015-08-31 09:00:00.0: BLMQI.01R1.B2I30_MQXA,BLMEL.01R1.B2I10_BPMSW.1R1,BLMQI.01R1.B1E10_MQXA,BLMQI.01R1.B2I20_MQXA,BLMQI.01R1.B1E20_MQXA,BLMQI.02R1.B2I30_MQXB,BLMQI.01R1.B1E30_MQXA,BLMQI.02R1.B2I23_MQXB,BLMQI.02R1.B1E21_MQXB,BLMQI.02R1.B2I22_MQXB
2016-04-28 14:57:37.0: BLMEL.01R1.B2I10_BPMSW.1R1,BLMQI.01R1.B2I30_MQXA,BLMQI.01R1.B1E10_MQXA,BLMQI.01R1.B2I20_MQXA,BLMQI.01R1.B1E20_MQXA,BLMQI.02R1.B2I30_MQXB,BLMQI.01R1.B1E30_MQXA,BLMQI.02R1.B2I23_MQXB,BLMQI.02R1.B1E21_MQXB,BLMQI.02R1.B2I22_MQXB
2016-09-16 02:00:00.0: BLMEL.01R1.B2I10_BPMSW.1R1,BLMQI.01R1.B2I30_MQXA,BLMQI.01R1.B1E10_MQXA,BLMQI.01R1.B2I20_MQXA,BLMQI.01R1.B1E20_MQXA,BLMQI.02R1.B2I30_MQXB,BLMQI.01R1.B1E30_MQXA,BLMQI.02R1.B2I23_MQXB,BLMQI.02R1.B1E21_MQXB,BLMQI.02R1.B2I22_MQXB
2016-09-16 02:00:01.0: BLMEL.01R1.B2I10_BPMSW.1R1,BLMQI.01R1.B2I30_MQXA,BLMQI.01R1.B1E10_MQXA,BLMQI.01R1.B2I20_MQXA,BLMQI.01R1.B1E20_MQXA,BLMQI.02R1.B2I30_MQXB,BLMQI.01R1.B1E30_MQXA,BLMQI.02R1.B2I23_MQXB,BLMQI.02R1.B1E21_MQXB,BLMQI.02R1.B2I22_MQXB
2016-09-16 19:16:30.0: BLMEL.01R1.B2I10_BPMSW.1R1,BLMQI.01R1.B2I30_MQXA,BLMQI.01R1.B1E10_MQXA,BLMQI.01R1.B2I20_MQXA,BLMQI.01R1.B1E20_MQXA,BLMQI.02R1.B2I30_MQXB,BLMQI.01R1.B1E30_MQXA,BLMQI.02R1.B2I23_MQXB,BLMQI.02R1.B1E21_MQXB,BLMQI.02R1.B2I22_MQXB
2017-01-01 01:00:00.0: BLMEL.01R1.B2I10_BPMSW.1R1,BLMQI.01R1.B2I30_MQXA,BLMQI.01R1.B1E10_MQXA,BLMQI.01R1.B2I20_MQXA,BLMQI.01R1.B1E20_MQXA,BLMQI.02R1.B2I30_MQXB,BLMQI.01R1.B1E30_MQXA,BLMQI.02R1.B2I23_MQXB,BLMQI.02R1.B1E21_MQXB,BLMQI.02R1.B2I22_MQXB