Merge branch 'mac-develop' into 'master'

Update Master with Basic Demo In place. Preparing for 1.0.0 release.

See merge request !5

.gitignore

@@ -20,6 +20,9 @@ bin/*
 .devcontainer/*
 *.code-workspace
 
+#For demos
+.ipynb_checkpoints/*
+
 #for leiningen builds
 .lein*
 /.lein-*

Makefile

@@ -29,6 +29,10 @@ JAR_INSTALL_SCRIPT=$(PROJECT_ROOT)/scripts/install/java/main/install.sh
 DOCS_SRC_DIR=$(PROJECT_ROOT)/build/docs/groovydoc
 API_DOCS_DIR=$(PROJECT_ROOT)/dist/docs/api
 
+TEMPLATE_SRC_DIR=$(PROJECT_ROOT)/resources/main/templates
+SESSION_TEMPLATE_SRC_DIR=$(TEMPLATE_SRC_DIR)/sessions
+METADATA_TEMPLATE_SRC_DIR=$(TEMPLATE_SRC_DIR)/metadata
+
 #C/C++ Related Paths
 C_MAIN_BUILD_DIR=$(PROJECT_ROOT)/scripts/build/c/main
 C_TEST_BUILD_DIR=$(PROJECT_ROOT)/scripts/build/c/test
@@ -38,15 +42,17 @@ TEST_RESOURCE_DIR=$(PROJECT_ROOT)/libs
 run-native-tests: export LD_LIBRARY_PATH = $(JAVA_HOME)/lib/jli:$(JAVA_HOME)/lib/server
 run-native-tests: export CLASSPATH = $(TEST_RESOURCE_DIR)/test/java/jars/$(SRC_JAR)
 
-.PHONY: build-c_cpp prep-java build-java dist-java run-native-tests copy-docs cleanup
+.PHONY: build-c_cpp prep-java build-java dist-java run-native-tests copy-docs create-demo-resources cleanup
 
-all: build-c_cpp prep-java build-java dist-java copy-docs run-native-tests cleanup
+all: build-c_cpp prep-java build-java dist-java copy-docs run-native-tests create-demo-resources cleanup
 
 build-c_cpp:
 	@echo "Building the shared library for C/C++ native sessions and the distributable native C artifacts."
 	@make -C $(C_MAIN_BUILD_DIR)
 	@echo "Building C/C++ binary for native transformation tests."
 	@make -C $(C_TEST_BUILD_DIR)/transformation
+	@echo "Building C/C++ binary for native condition tests."
+	@make -C $(C_TEST_BUILD_DIR)/condition
 	@echo "Building C/C++ binary for native endpoint tests."
 	@make -C $(C_TEST_BUILD_DIR)/endpoint
 	@echo "Building C/C++ binary for native session tests."
@@ -60,7 +66,7 @@ cleanup:
 prep-java:
 	@echo "Cleaning any residual Java artifacts."
 	@rm -rf $(JAVA_ARTIFACT_DIR)
-	@rm -rf $(API_DOCS_DIR)
+	@rm -rf $(API_DOCS_DIR)P
 	@echo "Java ready for build."
 
 build-java:
@@ -74,8 +80,10 @@ dist-java:
 	@echo "Creating distributable Java artifacts."
 	@cp $(JAR_SRC_DIR)/$(SRC_JAR) $(JAR_SRC_DIR)/$(TARGET_JAR)
 	@cp $(JAR_INSTALL_SCRIPT) $(JAR_SRC_DIR)
+	@cp $(SESSION_TEMPLATE_SRC_DIR)/*.json $(JAR_SRC_DIR)
+	@cp $(METADATA_TEMPLATE_SRC_DIR)/*.json $(JAR_SRC_DIR)
 	@mkdir -p $(JAVA_ARTIFACT_DIR)
-	@-cd $(JAR_SRC_DIR) && tar -cf $(LIBRARY_NAME).tar $(TARGET_JAR) install.sh
+	@-cd $(JAR_SRC_DIR) && tar -cf $(LIBRARY_NAME).tar $(TARGET_JAR) install.sh *.json
 	@-cd $(JAR_SRC_DIR) && mv $(LIBRARY_NAME).tar $(JAVA_ARTIFACT_DIR)
 
 copy-docs:
@@ -84,6 +92,10 @@ copy-docs:
 	@-cd $(DOCS_SRC_DIR) && tar -cf $(API_DOCS_DIR)/messageapi_docs.tar *
 	@echo "Finished copying API docs for distribution."
 
+create-demo-resources:
+	@echo "Creating Demo resources."
+	@echo "Finished creating Demo resources."
+
 run-native-tests:
 	@echo "Running native C/C++ session test."
 	@-cd $(TEST_RESOURCE_DIR)/test/c/session && ./SessionDemo.bin
@@ -103,4 +115,12 @@ install:
 	@chmod +x install.sh
 	@./install.sh "C_CPP"
 	@rm install.sh
-	@echo "Finished installing package, validate ~/.bashrc and env vars for success."
\ No newline at end of file
+	@echo "Finished installing package, validate ~/.bashrc and env vars for success."
+
+demo: install
+	@echo "Installing demo dependencies."
+	@echo "Finished insatlling demo dependencies."
+
+demo-k3: install-k3
+	@echo "Installing demo dependencies."
+	@echo "Finished insatlling demo dependencies."
\ No newline at end of file

README.md

@@ -14,7 +14,7 @@ Use of this package is described throughout this README through design discussio
 
 MessageAPI was built to promote and improve the long-term sustainability and enterprise-scale innovative capabilities of large research and operations organizations like NCEI, NESDIS, and NOAA. A simple and static (never changing) language-level API provided in multiple languages together with a completely configurable process configuration allows fast augmentation of legacy code to use evolutionary features, allows teams to reuse existing algorithms, and ensures that code-level details don't have to change as interface-level technologies evolve. The idea is to provide a capability so that Java/JVM, C/C++, Fortran, Python, R, and IDL programmers can continue to use their own preferred languages and technologies to do research and development, while ensuring that they can continue to evolve along with emerging organizational requirements and trends, without making code changes.
 
-MessageAPI has a small multi-language core that is easily installed on mission systems as a non-root user using the mature build system; allows whitelisting of executable resources and configuration-level exposition to maintain visible security; and provides a thread-safe, asynchronous, stream and batch capable data model to build composable, scalable, and orchestrated computational processes. 
+MessageAPI has a small multi-language core that is easily installed on mission systems as a non-root user using the mature build system; allows whitelisting of executable resources and configuration-level exposition to maintain visible security; and provides a thread-safe, asynchronous, stream and batch capable data model to build composable, scalable, and orchestrated computational processes.
 
 MessageAPI abstracts container factoring, container classification, conditional filtering, stateless transformation, and bidirectional endpoint processing into JSON configuration. This allows legacy code to be refactored or restructures in a fine-grained or large-scale way. Even the core parts of MessageAPI itself are configurable through JSON manifest, allowing the system to be customized to various use cases.
 
@@ -83,7 +83,7 @@ MessageAPI is designed to be capable of passing any message content - objects, d
 
 The benefits of using MessageAPI are its extreme schema flexibility, potential for tool decoupling and transport optimization, API standardization across all messaging, and self documentation through an information model.
 
-It's important to point out that even though it's called MessageAPI, the provided implementations do not replace message tools like ActiveMQ, RabbitMQ, Spark, or Storm. Really, no tools are 'replaced' by this one. MessageAPI is fundamentally a process specification, and the provided implementation is a small (3MB total package size) universal connector. This means that other messaging systems could of course be used with MessageAPI - linking response steps, serving as input or output queues, or transformation steps. Even MessageAPI sessions can be endpoints for MessageAPI sessions.
+It's important to point out that even though it's called MessageAPI, the provided implementations do not replace message tools like ActiveMQ, RabbitMQ, Spark, or Storm. Really, no tools are 'replaced' by this one. MessageAPI is fundamentally a process specification, and the provided implementation is a small (<1MB core package size) universal connector. This means that other messaging systems could of course be used with MessageAPI - linking response steps, serving as input or output queues, or transformation steps. Even MessageAPI sessions can be endpoints for MessageAPI sessions.
 
 ## Package Contents
 
@@ -564,6 +564,7 @@ or
 
 ```java
 import gov.noaa.messageapi.interfaces.ISession;
+import gov.noaa.messageapi.sessions.DefaultSession;
 
 ISession session = new DefaultSession("/path/to/session_manifest.json");
 ```
@@ -721,13 +722,14 @@ Because requests contain a copy of the session variables which created them, the
 ##### Package Use
 
 ###### Prebuilt Resources
+
 The package can be retrieved and used as a precompiled set of artifacts packaged as tar files. There are currently tar files for the core Java library and the precompiled C/C++ native library (compiled using the RHEL7 UBI, which is freely available from Redhat and compatible with Openshift). Each tar comes with an install.sh file that will install the relevant component for the current user. This means that for the Java package, the MessageAPI jar will be installed to a user directory, which will be added to the PATH and the Jar will be added to the CLASSPATH (if not already installed). The C library will add the shared library to the PATH and set up environment variables for use of the packaged header and source files. No root privileges are required to install or use the precompiled package resources. There is a central install script that will unpackage and install all components and relevant environment variables for you, for either "C_CPP" or "CORE". See below for instructions.
 
 To install on a mission system at NCEI, do the following:
 
 1. Make sure you are logged into the NCEI gitlab in the usual way. If you are logged, in, you will be able to access repositories.
 2. Copy/paste the following five lines in a terminal one by one, hitting enter after each, for the user that will be using MessageAPI:
-    - wget https://git.ncei.noaa.gov/sesb/sscs/messageapi/-/raw/master/scripts/install/package/install.sh?inline=false --no-check-certificate -O install.sh
+    - wget <https://git.ncei.noaa.gov/sesb/sscs/messageapi/-/raw/master/scripts/install/package/install.sh?inline=false> --no-check-certificate -O install.sh
     - chmod +x install.sh
     - ./install.sh "C_CPP"
     - rm install.sh
@@ -735,9 +737,10 @@ To install on a mission system at NCEI, do the following:
 3. You now have access to all MessageAPI resources needed to build and run MessageAPI dependent software, including build templates, headers, and shared libraries. Convenient environment variables have also been set up in the bashrc (which are also used by build templates) - use a cat ~/.bashrc to see which ones are available.
 
 ###### Building From Source
+
 When acquired through repository, the package can be built from source in order to run included tests. A Dockerfile for building the package is included in the resources/docker directory - this Dockerfile is based on the RHEL7 UBI and contains all necessary packages needed to build the MessageAPI system. This file can be used as-is or as a reference to guide what resources and conditions are necessary.
 
-Important to note - because of the way MessageAPI handles Session bootstrapping, it is not required or even desired to bundle the MessageAPI core with user classes. As long as these are made available to MessageAPI on the Java classpath at Session creation within a JAR, Sessions will be able to use them. This design makes it easier to automate things like creating K8s pods of certain session types. 
+Important to note - because of the way MessageAPI handles Session bootstrapping, it is not required or even desired to bundle the MessageAPI core with user classes. As long as these are made available to MessageAPI on the Java classpath at Session creation within a JAR, Sessions will be able to use them. This design makes it easier to automate things like creating K8s pods of certain session types.
 
 Also important to note for native Session use - MessageAPI Core and any user classes must be included in a JAR (not necessarily the same JAR), and these JARS must be explicitly listed on the Java CLASSPATH (they cannot be included  using directory wildcard expansions). This is due to how the JVM is created during session spin-up. See the CLASSPATH variable in ~/.bashrc with the #messageapi_core_set_classpath comment to see how the core is included on the classpath. To include user class-containing JARS, just add them in a similar way.
 
@@ -769,18 +772,16 @@ Other dependencies are installed for the purposes of running tests, including Gr
 
 ## Developer Guide
 
-In addition to reading issue, tag, and push history in the git repository, developers may refer to the more detailed [developer work log history](./docs/development/DeveloperWorkLog.md). This document outlines features currently and previously under focus, providing motivations, descriptions, design behaviors, and justifications.
+In addition to reading issue, tag, and push history in the git repository, developers may refer to the more detailed [developer work log history](./dist/docs/development/DeveloperWorkLog.md). This document outlines features currently and previously under focus, providing motivations, descriptions, design behaviors, and justifications.
 
 ### Bugs and Feature Requests
 
 The package is in current, active development, and as such, some bugs or desired features are expected. Either type of note can be sent on to ryan.berkheimer@noaa.gov.
 
-
 ### Caveats and Gotchas
 
 Important to note for native (non-JVM) Session use - MessageAPI Core and any user classes must be included in a JAR (not necessarily the same JAR), and these JARS must be explicitly listed on the Java CLASSPATH (they cannot be included  using directory wildcard expansions). This is due to how the JVM is created during session spin-up. See the CLASSPATH variable in ~/.bashrc with the #messageapi_core_set_classpath comment (after installing the package) to see how the core is included on the classpath. To include user class-containing JARS, just add them in a similar way.
 
-
 ## License
 
 THIS SOFTWARE AND ITS DOCUMENTATION ARE CONSIDERED TO BE IN THE PUBLIC

build.gradle

@@ -64,6 +64,7 @@ test {
     testLogging {
         showStandardStreams = true
     }
+    environment "MESSAGEAPI_SESSION_TEMPLATE_DIR", "/workspaces/messageapi/resources/test/standard_session"
 }
 
 

resources/demo/notebooks/.ipynb_checkpoints/.gitignore

+*
+!.gitignore
\ No newline at end of file

resources/demo/notebooks/C_Demo.ipynb

+%% Cell type:markdown id: tags:
+
+# Ensure Basic Kernel Operation
+
+%% Cell type:code id: tags:
+
+``` c
+#include <stdio.h>
+
+int main() {
+    printf("Hello world\n");
+}
+```
+
+%%%% Output: stream
+
+    Hello world
+
+%% Cell type:markdown id: tags:
+
+# Very Verbose File Reader
+
+%% Cell type:code id: tags:
+
+``` c
+//%cflags:-fPIC -std=c99 -I/home/messageapi/.messageapi/c/headers -I/usr/lib/jvm/java-11-openjdk-11.0.8.10-1.el7.x86_64/include -I/usr/lib/jvm/java-11-openjdk-11.0.8.10-1.el7.x86_64/include/linux -L/workspaces/messageapi/libs/test/c/session/so -lmessageapi
+#include <jni.h>
+#include <stdio.h>
+#include "messageapi_structs.h"
+#include "MessageApiSessionLib.h"
+
+int main()
+{
+    //puts("In our native session demo test!\n");
+    //puts("Hello, World\n");
+    //fflush(stdout);
+
+    session *session = createSession("/workspaces/messageapi/resources/test/file-reader-native/manifest.json");
+
+    //puts("Successfully created session.");
+    //fflush(stdout);
+
+    request *request1 = createRequest(session);
+
+    //puts("Successfully created a request.");
+    //fflush(stdout);
+
+    record *record1 = createRequestRecord(session, request1);
+
+    //puts("Successfully created a request record.");
+    //fflush(stdout);
+
+    field *field1 = getField(session, record1, "file-path");
+
+    //puts("Successfully retrieved the file-path field from the request record.");
+    //fflush(stdout);
+
+    setFieldStringVal(session, field1, "/workspaces/messageapi/resources/test/inputs/file-reader/simpletextfile");
+
+    //puts(getFieldStringVal(session, field1));
+    //fflush(stdout);
+
+    //puts("Successfully set the field string value for the file-path field from the request record.");
+    //fflush(stdout);
+
+    //record_list *session_records = createRecordList(session);
+
+    //addRecord(session, session_records, record1);
+
+    //setRequestRecords(session, request1, session_records);
+
+    //record *record2 = createRequestRecord(session, request1);
+    //field *field2 = getField(session, record2, "file-path");
+    //setFieldStringVal(session, field2, "/workspaces/messageapi/resources/test/inputs/file-reader/proc_sm_gtsnp_data_ftp_CF6_cf6_20190506.txt");
+
+    //record_list *request_records = getRequestRecords(session, request1);
+    //printf("Request record count: %d\n", request_records->count);
+    //fflush(stdout);
+
+    response *response = submitRequest(session, request1);
+
+    //puts("Response completion status right after submission:");
+    //fputs(isComplete(session, response) ? "true" : "false", stdout);
+    //puts("");
+    //fflush(stdout);
+
+    //puts("Successfully submitted the request.");
+    //fflush(stdout);
+
+    //puts(getFieldStringVal(session, field1));
+    //fflush(stdout);
+
+    while (!isComplete(session, response)) {}
+
+    //puts("Successfully returned from the request.");
+    //fflush(stdout);
+
+    //puts(getFieldStringVal(session, field1));
+    //fflush(stdout);
+
+    record_list *response_records = getResponseRecords(session, response);
+    //rejection_list *response_rejections = getResponseRejections(session, response);
+    //printf("Record count: %d\n", response_records->count);
+    //printf("Rejection count: %d\n", response_rejections->count);
+
+    for (int j=0; j<response_records->count; j++) {
+        record *return_record = getRecord(session, response_records, j);
+        string_list *fieldIds = getFieldIds(session, return_record);
+        for (int i=0; i<fieldIds->count; i++) {
+            field *returnField = getField(session, return_record, fieldIds->strings[i]);
+            //printf("FieldType: %s\n", getFieldType(session,returnField));
+            //if (strcmp(getFieldType(session,returnField), "integer") == 0) {
+             //   printf("Field id: %s,  Field type: %d\n", fieldIds->strings[i], getFieldIntVal(session,returnField));
+            //} else
+            if (strcmp(getFieldType(session,returnField), "string") == 0) {
+                //printf("Field id: %s,  Field value: %s\n", fieldIds->strings[i], getFieldStringVal(session,returnField));
+                printf("%s\n", getFieldStringVal(session,returnField));
+            }
+        }
+    }
+    fflush(stdout);
+    releaseSession(session);
+
+
+
+}
+```
+
+%%%% Output: stream
+
+    Hi
+    There
+    User
+    I
+    Am
+    A
+    Test!
+
+%% Cell type:markdown id: tags:
+
+# File Reader, slimmed
+
+%% Cell type:code id: tags:
+
+``` c
+//%cflags:-std=c99 -I/home/messageapi/.messageapi/c/headers -I/usr/lib/jvm/java-11-openjdk-11.0.8.10-1.el7.x86_64/include -I/usr/lib/jvm/java-11-openjdk-11.0.8.10-1.el7.x86_64/include/linux -L/workspaces/messageapi/libs/test/c/session/so -lmessageapi
+#include <jni.h>
+#include <stdio.h>
+#include "messageapi_structs.h"
+#include "MessageApiSessionLib.h"
+
+int main() {
+    const char *manifest = "/workspaces/messageapi/resources/test/file-reader-native/manifest.json";
+    const char *textfile = "/workspaces/messageapi/resources/test/inputs/file-reader/simpletextfile";
+    session *session = createSession(manifest);
+    request *request = createRequest(session);
+    record *input_record = createRequestRecord(session, request);
+    setFieldStringVal(session, getField(session, input_record, "file-path"), textfile);
+    response *response = submitRequest(session, request);
+    while (!isComplete(session, response)) {}
+    record_list *response_records = getResponseRecords(session, response);
+    for (int j=0; j<response_records->count; j++) {
+        record *return_record = getRecord(session, response_records, j);
+        string_list *fieldIds = getFieldIds(session, return_record);
+        for (int i=0; i<fieldIds->count; i++) {
+            field *returnField = getField(session, return_record, fieldIds->strings[i]);
+            if (strcmp(getFieldType(session,returnField), "string") == 0) {
+                printf("%s\n", getFieldStringVal(session,returnField));
+            }
+        }
+    }
+    fflush(stdout);
+    releaseSession(session);
+}
+```
+
+%%%% Output: stream
+
+    Hi
+    There
+    User
+    I
+    Am
+    A
+    Test!
+
+%% Cell type:code id: tags:
+
+``` c
+```

resources/demo/notebooks/Java_Demo.ipynb

+%% Cell type:code id: tags:
+
+``` java
+List<String> added = %jars /workspaces/messageapi/build/libs/messageapi-core-1.0.0.jar
+```
+
+%% Cell type:code id: tags:
+
+``` java
+import gov.noaa.messageapi.interfaces.ISession;
+```
+
+%% Cell type:code id: tags:
+
+``` java
+import gov.noaa.messageapi.sessions.StandardSession;
+```
+
+%% Cell type:code id: tags:
+
+``` java
+ISession session = new StandardSession("/workspaces/messageapi/resources/test/native-condition/parameters.json");
+```
+
+%% Cell type:code id: tags:
+
+``` java
+import gov.noaa.messageapi.interfaces.*;
+```
+
+%% Cell type:code id: tags:
+
+``` java
+IRequest request = session.createRequest()
+```
+
+%% Cell type:code id: tags:
+
+``` java
+IRecord record = request.createRecord();
+IRecord record2 = request.createRecord();
+```
+
+%% Cell type:code id: tags:
+
+``` java
+record.setField("int-field", 1);
+record2.setField("int-field", 100);
+```
+
+%% Cell type:code id: tags:
+
+``` java
+record.getField("int-field").getValue()
+```
+
+%%%% Output: execute_result
+
+    1
+
+%% Cell type:code id: tags:
+
+``` java
+IResponse response = request.submit()
+```
+
+%% Cell type:code id: tags:
+
+``` java
+response.isComplete()
+```
+
+%%%% Output: execute_result
+
+    true
+
+%% Cell type:code id: tags:
+
+``` java
+response.getRecords().size()
+```
+
+%%%% Output: execute_result
+
+    3
+
+%% Cell type:code id: tags:
+
+``` java
+response.getRecords().forEach(r -> r.getFields().forEach(f->System.out.println(f.getId() + ", " + f.getValue())))
+```
+
+%%%% Output: stream
+
+    int-field, 100
+    int-field, 100
+    int-field, 1
+
+%% Cell type:code id: tags:
+
+``` java
+IResponse response2 = request.submit()
+```
+
+%% Cell type:code id: tags:
+
+``` java
+response2.isComplete()
+```
+
+%%%% Output: execute_result
+
+    true
+
+%% Cell type:code id: tags:
+
+``` java
+response2.getRecords().size()
+```
+
+%%%% Output: execute_result
+
+    3
+
+%% Cell type:code id: tags:
+
+``` java
+record.getField("int-field").getValue()
+```
+
+%%%% Output: execute_result
+
+    1
+
+%% Cell type:code id: tags:
+
+``` java
+record.getFields()
+```
+
+%%%% Output: execute_result
+
+    [gov.noaa.messageapi.fields.DefaultField@233d0729]
+
+%% Cell type:code id: tags:
+
+``` java
+record.getFields().forEach(f->System.out.println(f.getId() + ", " + f.getValue()))
+```
+
+%%%% Output: stream
+
+    int-field, 1
+
+%% Cell type:code id: tags:
+
+``` java
+response.getRecords().size()
+```
+
+%%%% Output: execute_result
+
+    3
+
+%% Cell type:code id: tags:
+
+``` java
+response.getRejections().size()
+```
+
+%%%% Output: execute_result
+
+    0
+
+%% Cell type:code id: tags:
+
+``` java
+response2.getRejections().size()
+```
+
+%%%% Output: execute_result
+
+    0
+
+%% Cell type:code id: tags:
+
+``` java
+ISession fileReaderSession = new StandardSession("/workspaces/messageapi/resources/test/file-reader/parameter_map_style.json")
+```
+
+%% Cell type:code id: tags:
+
+``` java
+IRequest fileReaderRequest = fileReaderSession.createRequest();
+```
+
+%% Cell type:code id: tags:
+
+``` java
+IRecord record = fileReaderRequest.createRecord();
+```
+
+%% Cell type:code id: tags:
+
+``` java
+record.getFields().forEach(f->System.out.println(f.getId() + ", " + f.getValue()))
+```
+
+%%%% Output: stream
+
+    file-path, null
+
+%% Cell type:code id: tags:
+
+``` java
+record.setField("file-path", "/workspaces/messageapi/build/resources/test/inputs/file-reader/proc_sm_gtsnp_data_ftp_CF6_cf6_20190506.txt");
+```
+
+%% Cell type:code id: tags:
+
+``` java
+IResponse fileReaderResponse = fileReaderRequest.submit();
+fileReaderResponse.isComplete();
+```
+
+%%%% Output: execute_result
+
+    false
+
+%% Cell type:code id: tags:
+
+``` java
+fileReaderResponse.isComplete();
+```
+
+%%%% Output: execute_result
+
+    false
+
+%% Cell type:code id: tags:
+
+``` java
+fileReaderResponse.getRecords().size();
+```
+
+%%%% Output: execute_result
+
+    79794
+
+%% Cell type:code id: tags:
+
+``` java
+fileReaderResponse.getRecords().get(100);
+```
+
+%%%% Output: execute_result
+
+    gov.noaa.messageapi.records.schema.SchemaRecord@3136b65f
+