option --percontribution now avoids contributions not in the specified project

Author: barny

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.0
+current_version = 0.11.1
 commit = True
 tag = True
 

README.md

@@ -8,7 +8,7 @@
 
  SPDX-License-Identifier: MIT
 
- version="0.10.0"
+ version="0.11.1"
 
 
 Introduction

elmclient/__meta__.py

@@ -9,7 +9,7 @@
 
 app = 'elmoslcquery'
 description = 'Commandline OSLC query for ELM'
-version = '0.10.0'
+version = '0.11.1'
 license = 'MIT'
 author_name = 'Ian Barnard'
 author_mail = '[email protected]'

elmclient/_project.py

@@ -200,6 +200,20 @@ def get_gc_contributions(self,gcuri):
         result = self.execute_get_json(self.reluri("gcsdk-api/flatListOfContributionsForGcHierarchy"), params={'configurationUri':gcuri,'include':'*'} )
 #        print( f"{result=}" )
         return result
+    
+    # get contributions and component details for just this app
+    def get_our_contributions(self, gcuri):
+        gc_contribs = self.get_gc_contributions(gcuri)
+        results = []
+        for contrib in gc_contribs['configurations']:
+            if contrib['configurationUri'].startswith(self.reluri()):
+                results.append( [contrib['configurationUri'],contrib['componentUri'] ] )
+#                print( f"Included {contrib['configurationUri']}" )
+            else:
+#                print( f"Skipped {contrib['configurationUri']}" )
+                pass
+#        print( f"{results=}" )
+        return results
 
     def set_local_config(self, name_or_uri, global_config_uri=None):
         if name_or_uri:
@@ -211,13 +225,11 @@ def set_local_config(self, name_or_uri, global_config_uri=None):
                 # find the contribution for this component
                 config_uri = None
                 for config in gc_contribs['configurations']:
-                    print( f"Checking {config=} for {self.project_uri=}" )
+#                    print( f"Checking {config=} for {self.project_uri=}" )
                     if config['componentUri'] == self.project_uri:
                         config_uri = config['configurationUri']
-                if config_uri is None:
-                    burp
             if not config_uri:
-                raise Exception('Cannot find configuration [%s] in project [%s]' % (name_or_uri, self.uri))
+                raise Exception( 'Cannot find configuration [%s] in project [%s]' % (name_or_uri, self.uri))
         else:
             config_uri = None
 

elmclient/_qm.py

@@ -153,11 +153,26 @@ def load_components_and_configurations(self,force=False):
             self._components[cu]['component'] = c
         return (ncomps, nconfs)
 
-    def get_local_config(self, name_or_uri):
-        for cu, cd in self._configurations.items():
-            logger.debug( f"{cu=} {cd=} {name_or_uri=}" )
-            if cu == name_or_uri or cd['name'] == name_or_uri:
-                return cu
+    def get_local_config(self, name_or_uri, global_config_uri=None):
+        if global_config_uri:
+            if not name_or_uri:
+                raise Exception( "can't find a local config in a GC if config name not provided" )
+            # gc and local config both specified - try to avoid loading all the local configs by using the gc tree to locate the local config
+            gc_contribs = self.get_gc_contributions(global_config_uri)
+            # find the contribution for this component
+            config_uri = None
+            for config in gc_contribs['configurations']:
+#                print( f"Checking {config=} for {self.project_uri=}" )
+                if config['componentUri'] == self.project_uri:
+                    config_uri = config['configurationUri']
+            if not config_uri:
+                raise Exception( 'Cannot find configuration [%s] in project [%s]' % (name_or_uri, self.uri))
+            return config_uri
+        else:
+            for cu, cd in self._configurations.items():
+                logger.debug( f"{cu=} {cd=} {name_or_uri=}" )
+                if cu == name_or_uri or cd['name'] == name_or_uri:
+                    return cu
         return None
 
     # load the typesystem using the OSLC shape resources

elmclient/_rm.py

@@ -188,7 +188,7 @@ def load_folders(self,name_or_uri=None,force=False):
         return None
 
     def load_components_and_configurations(self,force=False):
-        if self._components and self._configurations and not force:
+        if self._components and not force:
             return
         logger.info( f"load_components_and_configurations {self=} {self.is_optin=}" ) 
         self._components = {}
@@ -307,7 +307,6 @@ def load_components_and_configurations(self,force=False):
                     compu = component_el.get("{%s}resource" % rdfxml.RDF_DEFAULT_PREFIX["rdf"])
                     compx = self.execute_get_rdf_xml(compu, intent="Retrieve component definition to find all configurations", action="Retrieve each configuration")
                     comptitle = rdfxml.xmlrdf_get_resource_text(compx, './/dcterms:title')
-
                     confu = rdfxml.xmlrdf_get_resource_uri(compx, './/oslc_config:configurations')
                     self._components[compu] = {'name': comptitle, 'configurations': {}, 'confs_to_load': [confu]}
                     ncomps += 1
@@ -383,6 +382,7 @@ def load_components_and_configurations(self,force=False):
         return (ncomps, nconfs)
 
     def load_configs(self):
+        # load configurations
         while self._confs_to_load:
             confu = self._confs_to_load.pop()
             if not confu:
@@ -435,27 +435,40 @@ def load_configs(self):
                 self._confs_to_load.append( rdfxml.xmlrdf_get_resource_uri(confmember, './oslc_config:baselines') )
                 self._confs_to_load.append( rdfxml.xmlrdf_get_resource_uri(confmember, './rm_config:changesets') )
 
-    def get_local_config(self, name_or_uri):
+    def get_local_config(self, name_or_uri, global_config_uri=None):
         logger.info( f"GLC {self=} {name_or_uri=}" )
-        self.load_configs()
-        result = None
-        filter = None
-        if name_or_uri.startswith("S:"):
-            filter="Stream"
-            name_or_uri = name_or_uri[2:]
-        elif name_or_uri.startswith("B:"):
-            filter="Baseline"
-            name_or_uri = name_or_uri[2:]
-        elif name_or_uri.startswith("C:"):
-            filter="ChangeSet"
-            name_or_uri = name_or_uri[2:]
-        for cu, cd in self._configurations.items():
-            if filter and cd['conftype'] != filter:
-                continue
-            if cu == name_or_uri or cd['name'] == name_or_uri:
-                if result:
-                    raise Exception( f"Config {name_or_uri} isn't unique - you could try prefixing it with S: for stream, B: for baseline, or C: for changeset")
-                result = cu
+        if global_config_uri:
+            # gc and local config both specified - try to avoid loading all the local configs by using the gc tree to locate the local config
+            gc_contribs = self.get_gc_contributions(global_config_uri)
+            # find the contribution for this component
+            config_uri = None
+            for config in gc_contribs['configurations']:
+#                print( f"Checking {config=} for {self.project_uri=}" )
+                if config['componentUri'] == self.project_uri:
+                    config_uri = config['configurationUri']
+            if not config_uri:
+                raise Exception( 'Cannot find configuration [%s] in project [%s]' % (name_or_uri, self.uri))
+            result = config_uri
+        else:
+            self.load_configs()
+            result = None
+            filter = None
+            if name_or_uri.startswith("S:"):
+                filter="Stream"
+                name_or_uri = name_or_uri[2:]
+            elif name_or_uri.startswith("B:"):
+                filter="Baseline"
+                name_or_uri = name_or_uri[2:]
+            elif name_or_uri.startswith("C:"):
+                filter="ChangeSet"
+                name_or_uri = name_or_uri[2:]
+            for cu, cd in self._configurations.items():
+                if filter and cd['conftype'] != filter:
+                    continue
+                if cu == name_or_uri or cd['name'] == name_or_uri:
+                    if result:
+                        raise Exception( f"Config {name_or_uri} isn't unique - you could try prefixing it with S: for stream, B: for baseline, or C: for changeset")
+                    result = cu
         return result
 
     # for RM, load the typesystem using the OSLC shape resources listed for the Requirements and Requirements Collection creation factories

elmclient/examples/OSLCQUERY.md

@@ -75,7 +75,8 @@ usage: oslcquery [-h] [-f SEARCHTERMS] [-n NULL] [-o ORDERBY] [-p PROJECTNAME] [
                  [-F CONFIGURATION] [-G GLOBALCONFIGURATION] [-H SAVECONFIGS] [-I] [-J JAZZURL] [-L LOGLEVEL]
                  [-M MAXRESULTS] [-N] [-O OUTPUTFILE] [-P PASSWORD] [-Q] [-R] [-S] [-T] [-U USERNAME] [-V] [-W]
                  [-X XMLOUTPUTFILE] [-Y] [-Z PROXYPORT] [--nresults NRESULTS] [--compareresults COMPARERESULTS]
-                 [--pagesize PAGESIZE] [--typesystemreport TYPESYSTEMREPORT] [--cachedays CACHEDAYS] [-0 SAVECREDS]
+                 [--pagesize PAGESIZE] [--typesystemreport TYPESYSTEMREPORT] [--cachedays CACHEDAYS]
+                 [--saverawresults SAVERAWRESULTS] [--saveprocessedresults SAVEPROCESSEDRESULTS] [-0 SAVECREDS]
                  [-1 READCREDS] [-2 ERASECREDS] [-3 SECRET] [-4]
 
 Perform OSLC query on a Jazz application, with results output to CSV (and other) formats - use -h to get some basic
@@ -109,8 +110,8 @@ options:
                         Post-filter: A property name that must have a value for the resource to be included in the
                         results - you can specify this option more than once
   -A APPSTRINGS, --appstrings APPSTRINGS
-                        A comma-seperated list of apps, the query goes to the first entry, default "rm,gc,ccm". Each
-                        entry must be a domain or domain:contextroot e.g. rm or rm:rm1 - Default can be set using
+                        A comma-seperated list of apps, the query goes to the first entry, default "rm". Each entry
+                        must be a domain or domain:contextroot e.g. rm or rm:rm1 - Default can be set using
                         environemnt variable QUERY_APPSTRINGS
   -C COMPONENT, --component COMPONENT
                         The local component (optional, you *have* to specify the local configuration using -F)
@@ -126,7 +127,7 @@ options:
                         id, the full URI, or the config name (not implemented yet)
   -H SAVECONFIGS, --saveconfigs SAVECONFIGS
                         Name of CSV file to save details of the local project components and configurations
-  -I, --totalize        For all columns with multiple results, put in the total instead of the results
+  -I, --totalize        For any column with multiple results, put in the total instead of the results
   -J JAZZURL, --jazzurl JAZZURL
                         jazz server url (without the /jts!) default https://jazz.ibm.com:9443 - Default can be set
                         using environemnt variable QUERY_JAZZURL - defaults to https://jazz.ibm.com:9443 which DOESN'T
@@ -172,8 +173,12 @@ options:
                         Load the specified project/configuration and then produce a simple HTML type system report of
                         resource shapes/properties/enumerations to this file
   --cachedays CACHEDAYS
-                        The number of days for caching received data, default 1. To disable caching use -WW. To keep
+                        The number of days for caching received data, default 7. To disable caching use -WW. To keep
                         using a non-default cache period you must specify this value every time
+  --saverawresults SAVERAWRESULTS
+                        Save the raw results as XML to this path/file prefix - pages are numbered starting from 0000
+  --saveprocessedresults SAVEPROCESSEDRESULTS
+                        Save the processed results as JSON to this path/file
   -0 SAVECREDS, --savecreds SAVECREDS
                         Save obfuscated credentials file for use with readcreds, then exit - this stores jazzurl, jts,
                         appstring, username and password
@@ -184,10 +189,9 @@ options:
                         Wipe and delete obfuscated credentials file
   -3 SECRET, --secret SECRET
                         SECRET used to encrypt and decrypt the obfuscated credentials (make this longer for greater
-                        security) - required if using -0 or -1
+                        security) - only affects if using -0 or -1
   -4, --credspassword   Prompt user for a password to save/read obfuscated credentials (make this longer for greater
-                        security)
-```
+                        security)```
 
 
 BEFORE you start:
@@ -455,7 +459,11 @@ To find all artifacts in project/component in a specific module id 3892 modified
 
 To find all artifacts in project/component in a specific module id 3892 modified before a specific date `-q rm:module=~3892 and dcterms:modified<"2020-08-01T21:51:40.979Z"^^xsd:dateTime` - NOTE this is using the enhanced OSLC Query sytnax for finding an artifact by id using ~
 
-To totalize a column which has multiple results replacing them with a count of the number of results, which might be useful for example to get a count of artifacts in each module, use a query for modules `rdm_types:ArtifactFormat=jazz_rm:Module` and select `oslc_rm:uses` then use the -I option. Because this doesn't need name resolution which can slow the query and post-processing down when processing many modules, also use -R and -Q - for results from components across a GC `/gc/configuration/26` in GCM project `gcproj` the query looks like: `-s oslc_rm:uses,dcterms:title -q rdm_types:ArtifactFormat=jazz_rm:Module -G 26 -E gcproj` - the result is a spreadsheet containing the module URI, the module name and identifier and a column with the count of artifacts in the module.
+To totalize a column which has multiple results replacing them with a count of the number of results, which might be useful for example to get a count of artifacts in each module, use a query for modules `rdm_types:ArtifactFormat=jazz_rm:Module` and select `oslc_rm:uses` then use the -I option. Because this doesn't need name resolution which can slow the query and post-processing down when processing many modules, also use -R and -Q - for results from components across a GC `/gc/configuration/26` in GCM project `gcproj` the query looks like: `-s oslc_rm:uses,dcterms:title -q rdm_types:ArtifactFormat=jazz_rm:Module -G 26 -E gcproj` - the result is a spreadsheet containing the module URI, the module name and identifier and a column with the count of artifacts in the module. You can improve performance (assuming you don't need friendly names, and counting artifacts/module doesn't really need friendly names) by suppressing the reading of the type system using `-Q` and `-R`, and because the results will be large you may consider carefully whether to suppress paging by using `--pagesize 0`.
+
+If you have a project with many components and/or configurations, you may observe that startup is slow - this is because by default if you don't specify a component then oslcquery reads all the components in the project and all the configurations in each component so that it can find the configuration which might be in any component. As of version 0.10.0 you can reduce this cost in a few ways: - most basic is to specify a component using -C, when you do this oslcquery only reads the configurations for that component. The next is to specify a component using `-C` and GC as well - using `-E` and `-G` -and also as a local configuration using `-F` - `oslcquery` will find the contribution to the GC for that component. If you only specify a GC configuration (usinging `-E` and `-G`) then oslcquery won't read RM components or configurations and the query will work on the project returning results from all contributing componenets in the project.
+
+If there are very many results then paging can add significant cost - you may want to suppress paging using `--pagesize 0` but BE CAREFUL not to impact on other users!
 
 GCM basic usage
 ===============