Merge branch 'mac-develop' into 'develop'

Comprehensive build system is complete.

See merge request !2

.gitignore

@@ -6,14 +6,19 @@
 *.jnilib
 *.gch
 
+#Development
+.DS_Store
 
 #for gradle builds
 build/*
 .gradle/*
+libs/*
 
 #for visual-studio-code
 bin/*
 .vscode/*
+.devcontainer/*
+*.code-workspace
 
 #for leiningen builds
 .lein*

Makefile

-.PHONY : install
+#Primary Makefile for the MessageAPI package.
+#
+#This Makefile currently controls the build for:
+#
+#1. Java-core build, test, and subsequent artifact (versioned uberjar + install script) tar packaging
+#2. C/C++ API shared library build, test, and subsequent artifact (versioned .so + current full source/header set + install script)
+#tar packaging
+#3. Java API documentation artifact (html documents as tar) packaging
+#
+#Note that the package itself has an install.sh bash script ($PKG/scripts/install/package/install.sh)
+#that controls installation of specific libraries for any logged in user on the NCEI mission network.
 
-all: install
+#Library Name and Version
+LIBRARY_NAME=messageapi
+MAJOR_VERSION=1
+MINOR_VERSION=0
+PATCH_VERSION=0
+EXTENSION_VERSION=all
+
+#Root Path Computation
+PROJECT_ROOT:=$(PWD)
+
+#Java Related Paths
+JAR_SRC_DIR=$(PROJECT_ROOT)/build/libs
+SRC_JAR=$(LIBRARY_NAME)-$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)-$(EXTENSION_VERSION).jar
+JAVA_ARTIFACT_DIR=$(PROJECT_ROOT)/dist/artifacts/java/$(LIBRARY_NAME)_$(MAJOR_VERSION)_$(MINOR_VERSION)_$(PATCH_VERSION)
+TARGET_JAR=messageapi-core-$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION).jar
+JAR_INSTALL_SCRIPT=$(PROJECT_ROOT)/scripts/install/java/main/install.sh
+
+DOCS_SRC_DIR=$(PROJECT_ROOT)/build/docs/groovydoc
+API_DOCS_DIR=$(PROJECT_ROOT)/dist/docs/api
+
+#C/C++ Related Paths
+C_MAIN_BUILD_DIR=$(PROJECT_ROOT)/scripts/build/c/main
+C_TEST_BUILD_DIR=$(PROJECT_ROOT)/scripts/build/c/test
+
+TEST_RESOURCE_DIR=$(PROJECT_ROOT)/libs
+
+run-native-tests: export LD_LIBRARY_PATH = $(JAVA_HOME)/lib/jli:$(JAVA_HOME)/lib/server
+run-native-tests: export CLASSPATH = $(TEST_RESOURCE_DIR)/test/java/jars/$(SRC_JAR)
+
+.PHONY: build-c_cpp prep-java build-java dist-java run-native-tests copy-docs cleanup
+
+all: build-c_cpp prep-java build-java dist-java copy-docs run-native-tests cleanup
+
+build-c_cpp:
+	@echo "Building the shared library for C/C++ native sessions and the distributable native C artifacts."
+	@make -C $(C_MAIN_BUILD_DIR)
+	@echo "Building C/C++ binary for native transformation tests."
+	@make -C $(C_TEST_BUILD_DIR)/transformation
+	@echo "Building C/C++ binary for native endpoint tests."
+	@make -C $(C_TEST_BUILD_DIR)/endpoint
+	@echo "Building C/C++ binary for native session tests."
+	@make -C $(C_TEST_BUILD_DIR)/session
+	@echo "Finished all C/C++ related build tasks."
+
+cleanup:
+	@echo "Cleaning up test resources."
+	@rm -rf $(TEST_RESOURCE_DIR)
+
+prep-java:
+	@echo "Cleaning any residual Java artifacts."
+	@rm -rf $(JAVA_ARTIFACT_DIR)
+	@rm -rf $(API_DOCS_DIR)
+	@echo "Java ready for build."
+
+build-java:
+	@echo "Running java gradle build and tests for MessageAPI."
+	@gradle
+	@mkdir -p $(TEST_RESOURCE_DIR)/test/java/jars
+	@cp $(PROJECT_ROOT)/build/libs/$(SRC_JAR) $(TEST_RESOURCE_DIR)/test/java/jars
+	@echo "Finished java gradle build and tests for MessageAPI."
+
+dist-java:
+	@echo "Creating distributable Java artifacts."
+	@cp $(JAR_SRC_DIR)/$(SRC_JAR) $(JAR_SRC_DIR)/$(TARGET_JAR)
+	@cp $(JAR_INSTALL_SCRIPT) $(JAR_SRC_DIR)
+	@mkdir -p $(JAVA_ARTIFACT_DIR)
+	@-cd $(JAR_SRC_DIR) && tar -cf $(LIBRARY_NAME).tar $(TARGET_JAR) install.sh
+	@-cd $(JAR_SRC_DIR) && mv $(LIBRARY_NAME).tar $(JAVA_ARTIFACT_DIR)
+
+copy-docs:
+	@echo "Copying API docs for distribution."
+	@mkdir -p $(API_DOCS_DIR)
+	@-cd $(DOCS_SRC_DIR) && tar -cf $(API_DOCS_DIR)/messageapi_docs.tar *
+	@echo "Finished copying API docs for distribution."
+
+run-native-tests:
+	@echo "Running native C/C++ session test."
+	@-cd $(TEST_RESOURCE_DIR)/test/c/session && ./SessionDemo.bin
+	@echo "Finished running native C/C++ session test."
+
+install-k3:
+	@echo "Installing package to system for current user."
+	@wget https://k3.cicsnc.org/rberkheimer/messageapi/-/raw/mac-develop/scripts/install/package/install_k3.sh?inline=false --no-check-certificate -O install_k3.sh
+	@chmod +x install_k3.sh
+	@./install_k3.sh "C_CPP"
+	@rm install_k3.sh
+	@echo "Finished installing package, validate ~./bashrc and env vars for success."
 
 install:
-	echo "Running gradle build for MessageAPI."
-	${GRADLE}
-	echo "Finished gradle build for MessageAPI."
+	@echo "Installing package to system for current user."
+	@wget https://git.ncei.noaa.gov/sesb/sscs/messageapi/-/raw/master/scripts/install/package/install.sh?inline=false --no-check-certificate -O install.sh
+	@chmod +x install.sh
+	@./install.sh "C_CPP"
+	@rm install.sh
+	@echo "Finished installing package, validate ~/.bashrc and env vars for success."
\ No newline at end of file

README.md

@@ -2,9 +2,21 @@
 
 ## Version
 
-**0.9.0** (Pre-Release)
+**0.9.10** (Pre-Release)
 
-MessageAPI uses [Semantic Versioning 2.0.0](https://semver.org/) as its versioning strategy. The first official version will be released upon the implementation of the C native API. Subsequent minor versions will release the fortran, python, IDL, and R APIs.
+MessageAPI uses [Semantic Versioning 2.0.0](https://semver.org/) as its versioning strategy. The first official version will be released upon the implementation of the C native API and validation of the Java core. Subsequent minor versions will release the Fortran, Python, R, and IDL APIs, and Bash CLI.
+
+## API Docs
+
+Use of this package is described throughout this README through design discussions and examples. Once familiar with the general use case of the package, more complete [API Docs](./dist/docs/api/messageapi_docs.tar) can be downloaded and referenced (open index.html after extracting the tar).
+
+## Executive Summary
+
+MessageAPI was built to promote and improve the long-term sustainability and enterprise-scale innovative capabilities of large research and operations organizations like NCEI, NESDIS, and NOAA. A simple and static (never changing) language-level API provided in multiple languages together with a completely configurable process configuration allows fast augmentation of legacy code to use evolutionary features, allows teams to reuse existing algorithms, and ensures that code-level details don't have to change as interface-level technologies evolve. The idea is to provide a capability so that Java/JVM, C/C++, Fortran, Python, R, and IDL programmers can continue to use their own preferred languages and technologies to do research and development, while ensuring that they can continue to evolve along with emerging organizational requirements and trends, without making code changes.
+
+MessageAPI has a small multi-language core that is easily installed on mission systems as a non-root user using the mature build system; allows whitelisting of executable resources and configuration-level exposition to maintain visible security; and provides a thread-safe, asynchronous, stream and batch capable data model to build composable, scalable, and orchestrated computational processes. 
+
+MessageAPI abstracts container factoring, container classification, conditional filtering, stateless transformation, and bidirectional endpoint processing into JSON configuration. This allows legacy code to be refactored or restructures in a fine-grained or large-scale way. Even the core parts of MessageAPI itself are configurable through JSON manifest, allowing the system to be customized to various use cases.
 
 ## Overview
 
@@ -67,8 +79,6 @@ Figure 1 below illustrates the layout of a Session - how MessageAPI structures a
 
 Everything that passes data in MessageAPI consumes and produces the same object type - the record. The manifests are used in code by creating a session based on them - and then using a stable API to create a request, add records, set field and/or condition values, and then submit, which immediately returns an async response. The response will hold its own records and rejections, and eventually get an 'isComplete' flag. That's the entirety of the API surface. Everything else depends on the Manifest.
 
-All of these actions are
-
 MessageAPI is designed to be capable of passing any message content - objects, database entries, emails, etc., to/through anywhere - database tables, inboxes, s3 buckets, ftp, smtp, kafka, files, directories etc.
 
 The benefits of using MessageAPI are its extreme schema flexibility, potential for tool decoupling and transport optimization, API standardization across all messaging, and self documentation through an information model.
@@ -133,7 +143,7 @@ The following is an example of a DefaultSession manifest:
 }
 ```
 
-The general pattern in a manifest is to declare a key with the class as the plugin, and then provide another map that holds constructor parameters. The specification for MessageAPI consists of
+The general pattern in a manifest is to declare a key with the class as the plugin, and then provide another map that holds constructor parameters. The specification for MessageAPI consists of:
 
 - A **Session**, which holds the primary **Schema**, **Container**, and **Protocol** dimensions
 - A **Schema**, which defines all the **Fields** and **Conditions** (optional) a user will interact with for each **Record**
@@ -164,10 +174,10 @@ Note that both Conditions and Transformations specify 'factories'. These factori
                      "fields": ["file-path"],
                      "conditions": ["is-relative-path"]}],
     "transformations": [{"id": "trans-1",
-                         "operator": "fix-relative-paths",
-                         "constructor": {"transform-key": "file-collection"},
-                         "records": {"file-collection": {"COLLECTION": "coll-1"}},
-                         "fields": ["file-path"]}],
+                         "operator": "gov.noaa.messageapi.test.transformations.FixRelativePathsTransformation",
+                         "constructor": {"transform-key": "file-collection",
+                                        "fields": ["file-path"]},
+                         "records": {"file-collection": {"COLLECTION": "coll-1"}}}],
     "connections": [{"id": "conn-1",
                      "transformations": ["trans-1"],
                      "constructor": {"file-fields": "file-path"},
@@ -283,7 +293,7 @@ In the provided implementation, the types understood by the system (for use by c
 
 ###### Condition Sets
 
-Conditions can be set on fields that qualify records when passed in, serving as a potentially powerful filtering tool. In the provided default plugin, conditions must specify at least a unique id, a type, and an operator. Conditions that are to be used in filtering must be valued on individual records. Conditions are also provided on a whole-request basis - to be used in containerization of records. In this case, conditions can be given values when
+Conditions can be set on fields that qualify records when passed in, serving as a potentially powerful dynamic filtering tool. In the provided default plugin, conditions must specify at least a unique id, a type, and an operator. Conditions that are to be used in filtering must be valued on individual records. Conditions are also provided on a whole-request basis - to be used in containerization of records. In this case, conditions can be given values on the request itself, to be used for parsing into collection-containers by applying to every potential record that is declared as part of a collection set.
 
 There are two condition types in the provided implementation  - composite and comparison.
 
@@ -419,7 +429,7 @@ There are three types of container groupings provided by the default MessageAPI
 
 - **Transformations** are computational containers. They are an optional component of MessageAPI. Transformations act as instances of immutable functions that can hold global, configurable constructor/initialization parameters, use other arbitrary named containers as arguments in a process function, return a list of records, and list the fields that each record will provide in a 'fields' map entry. Transformations are always immutable, executed lazily (only when called in an Endpoint), and always operate on and produce lists of records.
 
-The 'records' entry on the Transformation map itself holds a map of named parameter keys that correspond to values describing what type of container and the id of that container to use as that particular input. For example, in the following Transformation map, the first Transformation is called 'join-test'. The Transformation has an operator of 'join', which corresponds to a java class in the TransformationFactory listed in the Session manifest. This Transformation provides a constructor map containing two keys, 'join_field' and 'collection_field', both which specify constants that are set in the Transformation when it is initialized.
+The 'records' entry on the Transformation map itself holds a map of named parameter keys that correspond to values describing what type of container and the id of that container to use as that particular input. For example, in the following Transformation map, the first Transformation is called 'join-test'. The Transformation has an operator of the fully qualified join Class listed in the Session manifest. This Transformation provides a constructor map containing two keys, 'join_field' and 'collection_field', both which specify constants that are set in the Transformation when it is initialized.
 
 The 'join-test' Transformation defines three parameters in its process method map - 'parent', 'child', and 'other'. Each of these corresponds to a different container - when used in the Transformation, the 'parent' parameter will provide a list of all records contained by the 'namespace=condition-test' classifier; the 'child' parameter will provide a list of records contained by the 'mix-and-match' collection; and the 'other' parameter contains a special case - the UUID - which will provide every collection for a given UUID, and map the Transformation to every UUID, and all returned records for every UUID in this Transformation will be merged on return. The use of UUID as a Transformation record parameter is a special case, and due to its mapping ability, limits its use to once per Transformation.
 
@@ -434,21 +444,21 @@ It is the view of the package authors that Transformations hold computational ta
     "transformations": [
     {
         "id": "join-test",
-        "operator": "join",
+        "operator": "gov.noaa.messageapi.transformations.joins.StringFieldJoin",
         "constructor": {"join_field": "key",
-                        "collection_field": "mix-and-match"},
+                        "collection_field": "mix-and-match",
+                        "fields": ["key", "record", "filename", "type", "mix-and-match"]},
         "records": {"parent": {"CLASSIFIER": ["namespace", "condition-test"]},
                     "child":  {"COLLECTION": "mix-and-match"},
-                    "other": "UUID"},
-        "fields": ["key", "record", "filename", "type", "mix-and-match"]
+                    "other": "UUID"}
     },
     {
         "id": "reduce-test",
-        "operator": "reduce-sum",
+        "operator": "gov.noaa.messageapi.transformations.reductions.ReduceTransformation",
         "constructor": {"reduce-field": "mix-and-match",
-                        "reduce-target": "mix-and-match-reduction"},
-        "records": {"reduce-list" : {"TRANSFORMATION": "join-test"}},
-        "fields": ["key", "mix-and-match-reduction"]
+                        "reduce-target": "mix-and-match-reduction",
+                        "fields": ["key", "mix-and-match-reduction"]},
+        "records": {"reduce-list" : {"TRANSFORMATION": "join-test"}}
     }]
 }
 ```
@@ -595,9 +605,9 @@ Once the records are set, call submit on the request. This submission immediatel
 ```java
 import gov.noaa.messageapi.interfaces.IResponse;
 
-IResponse response = request.submit(); \\asynchronous, returns immediately
+IResponse response = request.submit(); //asynchronous, returns immediately
 
-System.out.println(response.isComplete()); \\would initially return 'False'
+System.out.println(response.isComplete()); //would initially return 'False'
 ```
 
 When complete, the IResponse will flip isComplete() to true, and will have available IRecords and IRejections.
@@ -710,7 +720,26 @@ Because requests contain a copy of the session variables which created them, the
 
 ##### Package Use
 
-When installed from source or acquired through repository, the MessageAPI-all.jar contains all dependencies required to run standalone. Because of the way MessageAPI handles Session bootstrapping, it is not required to bundle the MessageAPI core with user ConditionFactories, TransformationFactories, or Endpoints. As long as these are made available to MessageAPI on the Java classpath at Session creation, Sessions will be able to use them. This design makes it easier to automate things like creating K8s pods of certain session types.
+###### Prebuilt Resources
+The package can be retrieved and used as a precompiled set of artifacts packaged as tar files. There are currently tar files for the core Java library and the precompiled C/C++ native library (compiled using the RHEL7 UBI, which is freely available from Redhat and compatible with Openshift). Each tar comes with an install.sh file that will install the relevant component for the current user. This means that for the Java package, the MessageAPI jar will be installed to a user directory, which will be added to the PATH and the Jar will be added to the CLASSPATH (if not already installed). The C library will add the shared library to the PATH and set up environment variables for use of the packaged header and source files. No root privileges are required to install or use the precompiled package resources. There is a central install script that will unpackage and install all components and relevant environment variables for you, for either "C_CPP" or "CORE". See below for instructions.
+
+To install on a mission system at NCEI, do the following:
+
+1. Make sure you are logged into the NCEI gitlab in the usual way. If you are logged, in, you will be able to access repositories.
+2. Copy/paste the following five lines in a terminal one by one, hitting enter after each, for the user that will be using MessageAPI:
+    - wget https://git.ncei.noaa.gov/sesb/sscs/messageapi/-/raw/master/scripts/install/package/install.sh?inline=false --no-check-certificate -O install.sh
+    - chmod +x install.sh
+    - ./install.sh "C_CPP"
+    - rm install.sh
+    - source ~/.bashrc
+3. You now have access to all MessageAPI resources needed to build and run MessageAPI dependent software, including build templates, headers, and shared libraries. Convenient environment variables have also been set up in the bashrc (which are also used by build templates) - use a cat ~/.bashrc to see which ones are available.
+
+###### Building From Source
+When acquired through repository, the package can be built from source in order to run included tests. A Dockerfile for building the package is included in the resources/docker directory - this Dockerfile is based on the RHEL7 UBI and contains all necessary packages needed to build the MessageAPI system. This file can be used as-is or as a reference to guide what resources and conditions are necessary.
+
+Important to note - because of the way MessageAPI handles Session bootstrapping, it is not required or even desired to bundle the MessageAPI core with user classes. As long as these are made available to MessageAPI on the Java classpath at Session creation within a JAR, Sessions will be able to use them. This design makes it easier to automate things like creating K8s pods of certain session types. 
+
+Also important to note for native Session use - MessageAPI Core and any user classes must be included in a JAR (not necessarily the same JAR), and these JARS must be explicitly listed on the Java CLASSPATH (they cannot be included  using directory wildcard expansions). This is due to how the JVM is created during session spin-up. See the CLASSPATH variable in ~/.bashrc with the #messageapi_core_set_classpath comment to see how the core is included on the classpath. To include user class-containing JARS, just add them in a similar way.
 
 If desired or needed, MessageAPI can be bundled into other JARS or packages containing user Factories and Endpoints for portability, performance, security, or other reasons.
 
@@ -720,15 +749,15 @@ If desired or needed, MessageAPI can be bundled into other JARS or packages cont
 
 ## Installation and Deployment
 
-At the time of this writing, MessageAPI (0.0.10-PRERELEASE) was built using OpenJDK 11.0.3 with gradle 5.4.1. Older JDK versions are not guaranteed to work.
+At the time of this writing, MessageAPI was built and tested using OpenJDK 1.8 as well as 11.0.3 with gradle 5.4.1. Older JDK versions are not guaranteed to work.
 
 There was a breaking change between older versions of gradle and the 5 series, and a relative path resolution
 method was updated to accommodate this change. If building from scratch, the gradle version must be upgraded to 5.4.1+.
 
-Once these two system dependencies are met, this package can be run with tests by running 'gradle' from the package root.
+Once these two system dependencies are met, this package can be run with tests by running 'make' from the package root.
 
 ```Makefile
-gradle
+make
 ```
 
 If tests complete successfully, gradle will install MessageAPI to the local repository as an UberJAR on disk (usually in ~/.m2). UberJARs contain all of their dependencies, so the package can be run, for example, in a Java-enabled JupyterNotebook Kernel. Gradle will also create the javadocs (groovydocs) in the build directory ($PACKAGE/build/docs/groovydoc/index.html).
@@ -740,12 +769,17 @@ Other dependencies are installed for the purposes of running tests, including Gr
 
 ## Developer Guide
 
-In addition to reading issue, tag, and push history in the git repository, developers may refer to the more detailed [developer work log history](./DeveloperWorkLog.md). This document outlines features currently and previously under focus, providing motivations, descriptions, design behaviors, and justifications.
+In addition to reading issue, tag, and push history in the git repository, developers may refer to the more detailed [developer work log history](./docs/development/DeveloperWorkLog.md). This document outlines features currently and previously under focus, providing motivations, descriptions, design behaviors, and justifications.
+
+### Bugs and Feature Requests
+
+The package is in current, active development, and as such, some bugs or desired features are expected. Either type of note can be sent on to ryan.berkheimer@noaa.gov.
+
+
+### Caveats and Gotchas
 
-### Bugs
+Important to note for native (non-JVM) Session use - MessageAPI Core and any user classes must be included in a JAR (not necessarily the same JAR), and these JARS must be explicitly listed on the Java CLASSPATH (they cannot be included  using directory wildcard expansions). This is due to how the JVM is created during session spin-up. See the CLASSPATH variable in ~/.bashrc with the #messageapi_core_set_classpath comment (after installing the package) to see how the core is included on the classpath. To include user class-containing JARS, just add them in a similar way.
 
-The package is in current, active development.
-All bugs encountered should be reported to ryan.berkheimer@noaa.gov.
 
 ## License
 

build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'groovy'
     id 'maven-publish'
     id 'java'
-    id 'com.github.johnrengelman.shadow' version '5.0.0'
+    id 'com.github.johnrengelman.shadow' version '5.2.0'
 }
 
 shadowJar {
@@ -15,6 +15,7 @@ shadowJar {
     }
 }
 
+
 publishing {
   publications {
     maven(MavenPublication) {
@@ -39,21 +40,27 @@ publishing {
 sourceCompatibility = "${javaVersion}"
 targetCompatibility = "${javaVersion}"
 
-defaultTasks 'clean', 'rebuild','uploadShadow', 'publishToMavenLocal', 'groovydoc'
+defaultTasks 'clean', 'rebuild', 'uploadShadow', 'publishToMavenLocal', 'groovydoc'
+
+
+configurations {
+    // configuration that holds jars to include in the jar
+    directDeps
+}
 
 repositories {
     mavenCentral()
     jcenter()
 }
 
-
 dependencies {
-    compile ("com.googlecode.json-simple:json-simple:${jsonSimpleVersion}",
-             "commons-net:commons-net:${apacheCommonsNetVersion}",
+    directDeps ("com.googlecode.json-simple:json-simple:${jsonSimpleVersion}")
+    compile ("commons-net:commons-net:${apacheCommonsNetVersion}",
              "org.codehaus.groovy:groovy-all:${groovyVersion}")
     testCompile ("org.slf4j:slf4j-simple:${slf4jVersion}",
                 "org.spockframework:spock-core:${spockVersion}")
     testRuntime ("com.athaydes:spock-reports:${spockReportsVersion}")
+    configurations.compile.extendsFrom(configurations.directDeps)
 }
 
 jar {
@@ -61,6 +68,11 @@ jar {
         attributes 'Implementation-Title': 'MessageAPI',
                    'Implementation-Version': "${messageApiVersion}"
     }
+    /*from {
+        configurations.directDeps.collect { 
+            it.isDirectory() && it.getPath().contains("json-simple") ? it : project.logger.lifecycle(it.getPath()) 
+        }
+    }*/
 }
 
 sourceSets {

DeveloperWorkLog.md → dist/docs/development/DeveloperWorkLog.md

@@ -9,19 +9,24 @@ with current development, and should be supplemental to the repository record.
 
 This work log is also useful as a supplemental documentation for end users of the package - features are described and contextualized with as much detail as possible.
 
-## Future Work
+## Future Work in MessageAPI Core
 
 - Fortran API
 - Python API
-- Split Publish/Consumer Sessions
-- Example Endpoints (Email, Directory, Kafka)
+- R API
+- IDL API
+- Shell Utilities
 
 ## Current Focus
 
 ### C API
 
-#### Status - In Progress
+#### Status - In Resolution
 
+07/15/2020
+At this stage - the entire API is complete. The general structure is to have importable libs for each interface component (transformation, endpoint, session, condition) and have these libs contain access to every piece of processing logic that users might need. For each lib is associated
+
+06/01/2020
 We are currently working on the linkage between C++ and the NativeEndpoint class. The NativeEndpoint class has been established and the majority of C structs have been created. The next challenge is to create C/C++ methods that work on these C structs, and then methods that bind to Java methods.
 
 #### Description and Goals

gradle.properties

 #Software version
-messageApiVersion = 0.0.10
+messageApiVersion = 1.0.0
 
 #Dependency versions
 
@@ -7,11 +7,11 @@ messageApiVersion = 0.0.10
 #javaVersion = 11.0.3
 javaVersion = 1.8
 
-#Utils
+#Runtime Deps
 jsonSimpleVersion = 1.1.1
 
 #Build
-shadowVersion = 2.0.2
+shadowVersion = 5.2.0
 
 #Testing
 slf4jVersion = 1.7.13

messageapi-core.code-workspace

+{
+	"folders": [
+		{
+			"path": "."
+		}
+    ],
+    "settings": {
+        "java.configuration.updateBuildConfiguration": "automatic",
+        "files.associations": {
+            "gov_noaa_messageapi_endpoints_nativeendpoint.h": "c",
+            "*.tcc": "cpp",
+            "cctype": "cpp",
+            "clocale": "cpp",
+            "cmath": "cpp",
+            "complex": "cpp",
+            "cstdarg": "cpp",
+            "cstdint": "cpp",
+            "cstdio": "cpp",
+            "cstdlib": "cpp",
+            "cwchar": "cpp",
+            "cwctype": "cpp",
+            "exception": "cpp",
+            "system_error": "cpp",
+            "type_traits": "cpp",
+            "initializer_list": "cpp",
+            "iosfwd": "cpp",
+            "iostream": "cpp",
+            "istream": "cpp",
+            "limits": "cpp",
+            "new": "cpp",
+            "ostream": "cpp",
+            "sstream": "cpp",
+            "stdexcept": "cpp",
+            "streambuf": "cpp",
+            "cinttypes": "cpp",
+            "typeinfo": "cpp",
+            "string": "cpp",
+            "*.jnilib": "c"
+        }
+    }
+}
\ No newline at end of file

resources/docker/Dockerfile.develop

+#Dockerfile for RHEL7 MessageAPI Build on UBI (Openshift Compatible)
+#This dockerfile installs the system so that it can be interactively developed and built.
+#All build tools are installed and paths are set properly. Some probing tools are also
+#installed automatically for development.
+
+#Base Image (RHEL7 UBI)
+FROM registry.access.redhat.com/ubi7/ubi:latest
+
+#Install standard RHEL7 tools
+RUN yum -y install --disableplugin=subscription-manager \
+    locate vim unzip wget git java-11-openjdk-devel gcc gcc-gfortran gcc-c++ make \
+    && yum --disableplugin=subscription-manager clean all
+
+#Install Java Variables
+RUN cd $(dirname $(readlink -f $(which java))) && cd .. \
+    && export JAVA_HOME=$PWD \
+    && touch /etc/profile.d/java.sh \
+    && echo "export JAVA_HOME=${JAVA_HOME}" > /etc/profile.d/java.sh \
+    && echo "export PATH=${JAVA_HOME}/bin:$PATH" >> /etc/profile.d/java.sh \
+    && echo "export CLASSPATH=${JAVA_HOME}/bin:${CLASSPATH}" >> /etc/profile.d/java.sh
+
+#Install Gradle and Variables
+RUN wget https://services.gradle.org/distributions/gradle-5.4.1-bin.zip -P /tmp \
+    && unzip -d /opt/gradle /tmp/gradle-*.zip \
+    && export GRADLE_HOME=/opt/gradle/gradle-5.4.1 \
+    && touch /etc/profile.d/gradle.sh \
+    && echo "export GRADLE_HOME=${GRADLE_HOME}" > /etc/profile.d/gradle.sh \
+    && echo "export PATH=${GRADLE_HOME}/bin:$PATH" >> /etc/profile.d/gradle.sh
+
+#Add default type user 'messageapi'
+RUN useradd messageapi
+
+#Change to new 'messageapi' user
+USER messageapi
+
+#Set PATH and other user variables in bashrc
+RUN cd $(dirname $(readlink -f $(which java))) && cd .. \
+    && export JAVA_HOME=$PWD \
+    && echo "PATH=${JAVA_HOME}/bin:\$PATH" >> ~/.bashrc \
+    && export GRADLE_HOME=/opt/gradle/gradle-5.4.1 \
+    && echo "PATH=${GRADLE_HOME}/bin:\$PATH" >> ~/.bashrc \
+    && source ~/.bashrc
\ No newline at end of file

.gitlab-ci.yml → resources/gitlab/.gitlab-ci.yml

-#Testing 123456789012345
+#Gitlab Runner Configuration for Automatic CI/CD
 stages:
   - test
   - build

resources/test/basic-native/parameters.json

@@ -27,10 +27,11 @@
                         "fields": ["initial-value", "string-test", "int-list-test"]}],
     "connections": [{"id": "1",
                      "collections": ["coll-1"],
-                     "constructor": {"native-library": "/Users/rberkheimer/projects/asos/ingest/libraries/messageapi/lib/test/native/demoendpointlibrary/libDemoEndpointLibrary.jnilib",
+                     "constructor": {"native-library": "/workspaces/messageapi/libs/test/c/endpoint/libEndpointDemo.so",
                                     "default-fields": [{"id": "test-integer",
                                                         "type": "integer",
-                                                        "required": false},
+                                                        "required": false,
+                                                        "value": 5},
                                                        {"id": "return-list",
                                                         "type": "list(string)",
                                                         "required": false}],

resources/test/containers/simple/clisam/transformations.json

@@ -2,19 +2,28 @@
     "transformations": [
     {
         "id": "join-test",
-        "operator": "join",
+        "operator": "gov.noaa.messageapi.transformations.joins.StringFieldJoin",
         "constructor": {"join-field": "key",
-                        "collection-field": "mix-and-match"},
+                        "collection-field": "mix-and-match",
+                        "fields": [
+                            "key",
+                            "record",
+                            "filename",
+                            "type",
+                            "mix-and-match"
+                        ]},
         "records": {"parent": {"CLASSIFIER": ["namespace", "condition-test"]},
-                    "child":  {"COLLECTION": "mix-and-match"}},
-        "fields": ["key", "record", "filename", "type", "mix-and-match"]
+                    "child":  {"COLLECTION": "mix-and-match"}}
     },
     {
         "id": "reduce-test",
-        "operator": "reduce-sum",
+        "operator": "gov.noaa.messageapi.transformations.reductions.ReduceTransformation",
         "constructor": {"reduce-field": "mix-and-match",
-                        "reduce-target": "mix-and-match-reduction"},
-        "records": {"reduce-list" : {"TRANSFORMATION": "join-test"}},
-        "fields": ["key", "mix-and-match-reduction"]
+                        "reduce-target": "mix-and-match-reduction",
+                        "fields":[
+                            "key",
+                            "mix-and-match-reduction"
+                        ]},
+        "records": {"reduce-list" : {"TRANSFORMATION": "join-test"}}
     }]
 }

resources/test/containers/simple/condition-test/transformations.json

@@ -2,19 +2,28 @@
     "transformations": [
     {
         "id": "join-test",
-        "operator": "join",
+        "operator": "gov.noaa.messageapi.transformations.joins.StringFieldJoin",
         "constructor": {"join-field": "key",
-                        "collection-field": "mix-and-match"},
+                        "collection-field": "mix-and-match",
+                        "fields": [
+                            "key",
+                            "record",
+                            "filename",
+                            "type",
+                            "mix-and-match"
+                        ]},
         "records": {"parent": {"CLASSIFIER": ["namespace", "condition-test"]},
-                    "child":  {"TRANSFORMATION": "reduce-test"}},
-        "fields": ["key", "record", "filename", "type", "mix-and-match"]
+                    "child":  {"TRANSFORMATION": "reduce-test"}}
     },
     {
         "id": "reduce-test",
-        "operator": "reduce",
+        "operator": "gov.noaa.messageapi.transformations.reductions.ReduceTransformation",
         "constructor": {"reduce-field": "mix-and-match",
-                        "reduce-target": "mix-and-match-reduction"},
-        "records": {"reduce-list" : "UUID"},
-        "fields": ["key", "mix-and-match-reduction"]
+                        "reduce-target": "mix-and-match-reduction",
+                        "fields": [
+                            "key",
+                            "mix-and-match-reduction"
+                        ]},
+        "records": {"reduce-list" : "UUID"}
     }]
 }