fixed using a specific config, build 0.27.0 - new eamples of dncompare, validate, trsreader (UNFINISHED!)

Incorporated suggestions from wulmer to allow use in docker PR #69
Also incorporated wulmer's PR #67

Author: barny

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.26.2
+current_version = 0.27.0
 commit = True
 tag = True
 

README.md

@@ -8,11 +8,17 @@
 
  SPDX-License-Identifier: MIT
 
- version="0.26.2"
+ version="0.27.0"
 
 What's New?
 ===========
 
+02-Dec-2024
+* Added examples dncompare, validate, trsreader (this is VERY UNFINISHED!) - see the code in the examples folder
+* Fixed oslcquery working from a specific config using -F
+* Fixed oslcquery when trying to select a scoped attribute like `oslc:rm:uses{dcterms:identifier}` wasn't working - note that DN doesn't appear to support retreiving primary text of all artifacts in a module using `oslc:rm:uses{jazz_rm:primaryText}` - text isn't returned
+* Some (unfinished!) internal preparation for demonstrator of the new in 7.1 type query capability https://jazz.net/wiki/bin/view/Main/DNGTypeAPI
+
 11-Dec-2023
 * Added _validate.py to provide the Validate API, available on all apps (may need to restrict this to just ccm/gc/qm/rm?)
 * Added examples/validate.py as an example of using the validate API - for most options refer to oslcquery readme
@@ -153,45 +159,34 @@ This code provides examples of using various ELM APIs:
 
 * DN
 
-** Process - `elmclient`
-
-** OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
-
-** Module Structure - `dn_simple_modulestructure.py` - currently external to `elmclient`
-
-** ReqIF - `reqif_io.py` - currently reqif API is external to `elmclient`
-
-** Reportable REST (incomplete for qm and ccm) - `represt.py` for user use, internally for each application in `_rm.py`, `_ccm.py`, `_qm.py` 
-
+  * Process - `elmclient`
+  * OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
+  * Module Structure - `dn_simple_modulestructure.py` - currently external to `elmclient`
+  * ReqIF - `reqif_io.py` - currently reqif API is external to `elmclient`
+  * Reportable REST (incomplete for qm and ccm) - `represt.py` for user use, internally for each application in `_rm.py`, `_ccm.py`, `_qm.py` 
 
 * ETM
 
-** Process - `elmclient`
-
-** OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
-
-** Reportable REST (incomplete for qm and ccm) - `represt.py` for user use, internally for each application in `_rm.py`, `_ccm.py`, `_qm.py` 
-
+  * Process - `elmclient`
+  * OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
+  * Reportable REST (incomplete for qm and ccm) - `represt.py` for user use, internally for each application in `_rm.py`, `_ccm.py`, `_qm.py` 
 
 * EWM
 
-** Process - `elmclient`
-
-** OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
-
-** Reportable REST (incomplete for qm and ccm) - `represt.py` for user use, internally for each application in `_rm.py`, `_ccm.py`, `_qm.py` 
+  * Process - `elmclient`
+  * OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
+  * Reportable REST (incomplete for qm and ccm) - `represt.py` for user use, internally for each application in `_rm.py`, `_ccm.py`, `_qm.py` 
 
 
 * GCM
 
-** Process - `elmclient`
-
-** OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
+  * Process - `elmclient`
+  * OSLC including OSLC Query - `oslcquery.py` for user use, internally `oslcqueryapi.py` implements OSLC Query parsing and querying
 
 
 Reporting issues, and contributing
 ==================================
 
 If you find a problem with elmclient you can report it on the github issues https://github.com/IBM/ELM-Python-Client/issues - note this is just for issues with elmclient code. All other issues will likely be closed immediately.
 
-You can do a pull request to propose updates - there's no guarantee of if/when/how these will be merged but we certainly hope to benefit from contributions!
+You can do a pull request to propose updates - there's no guarantee of if/when/how these will be merged but we certainly hope to benefit from contributions!

elmclient/__meta__.py

@@ -9,7 +9,7 @@
 
 app = 'elmoslcquery'
 description = 'Commandline OSLC query for ELM'
-version = '0.26.2'
+version = '0.27.0'
 license = 'MIT'
 author_name = 'Ian Barnard'
 author_mail = '[email protected]'

elmclient/_ccm.py

@@ -252,6 +252,7 @@ def is_type_uri(self, uri):
 
     def resolve_uri_to_name(self, uri, trytouseasid=False):
         logger.info( f"resolve_uri_to_name {uri=}" )
+#        return self.__super__.resolve_uri_to_name(self, uri, trytouseasid=True)
         if not uri:
             result = None
             return result

elmclient/_qm.py

@@ -107,7 +107,7 @@ def load_components_and_configurations(self,force=False):
 
             components_uri = rdfxml.xmlrdf_get_resource_uri(cmsp_xml, './/rdf:Description/rdf:type[@rdf:resource="http://open-services.net/ns/core#QueryCapability"]/../oslc:resourceType[@rdf:resource="http://open-services.net/ns/config#Component"]/../oslc:queryBase')
             logger.info( f"{components_uri=}" )
-            print( f"{components_uri=}" )
+#            print( f"{components_uri=}" )
             # get all components
             crx = self.execute_get_xml( components_uri, intent="Retrieve component definition" )
             logger.info( f"{crx=}" )

elmclient/_queryparser.py

@@ -244,7 +244,7 @@ def do_oslcand(self, s):
 
     def term(self, s):
         # if RHS (or for in any of the items in RHS list) is a valueidentifier without a prefix, this uses the lhs to resolve the valueidentifier, i.e. as context for attribute value names
-        # or if RHS is an untyped literal then LHS is used to appy ^^type:name to it to make it a typedliteral
+        # or if RHS is an untyped literal then LHS is used to apply ^^type:name to it to make it a typedliteral
         # check if first elem is a property identifier, and if so see if value(s) are identifiers if so resolve them in context of the first identifier (e.g. for enum values)
         logger.info( f"Term {type(s)} {s}" )
         identifier, op, value = s
@@ -539,10 +539,11 @@ def scoped_term(self, s):
 
 # from https://tools.oasis-open.org/version-control/svn/oslc-core/trunk/specs/oslc-core.html#selectiveProperties
 # with slight tweaks to implement identifier
+# NOTE that for oslc.select NAME can be terminated by { or } to allow for nested properties
 _select_grammar  ="""
 select_terms    : properties
 properties      : property ("," property)*
-property        : dottedname | identifier | wildcard | nested_prop
+property        : dottedname | nested_prop | identifier | wildcard 
 dottedname      : NAME "." NAME
 nested_prop     : (identifier | wildcard) "{" properties "}"
 wildcard        : "*"
@@ -551,7 +552,7 @@ def scoped_term(self, s):
                     | "'" SPACYNAME "'"
 
 URI_REF_ESC     : /<https?:.*>/
-NAME            : /[a-zA-Z0-9_][^, ]*/
+NAME            : /[a-zA-Z0-9_][^, {}]*/
 SPACYNAME           : /[a-zA-Z0-9_][^']*/
 """
 
@@ -583,13 +584,17 @@ def __init__(self, resolverobject):
         self.prefixes = {} # prefixes used (will have to be added to oslc.prefix) - NOTE the key is the uri, the value is the prefix!
 
     def select_terms(self,s):
+        logger.info( f"select_terms {s=} {s[0]=}" )
         return s[0]
 
     def select_term(self,s):
+        logger.info( f"select_term {s=} {s[0]=}" )
         return s
 
     def nested_prop( self,s):
+        logger.info( f"nested_prop {s=} {s[0]=}" )
         result = s[0]+"{"+",".join(s[1])+"}"
+        logger.info( f"nested_prop {result=}" )
         return result
 
     def wildcard(self,s):
@@ -623,15 +628,18 @@ def identifier(self, s):
         logger.info( f"identifier {s=} {s[0]=}" )
         if len(s) == 1:
             resultname = s[0].value
+            logger.info( f"{resultname=}" )
         elif len(s) > 1:
             # a prefixed name
             resultname = ":".join([s[0].value, s[1].value])
             if s[0].value in rdfxml.RDF_DEFAULT_PREFIX:
                 self.prefixes[rdfxml.RDF_DEFAULT_PREFIX[s[0].value]]=s[0].value
+                logger.info( f"Added prefix {s[0].value=}" )
             else:
                 raise Exception( f"Prefix in orderby '{s[0].value}' not found!" )
         # look it up and if necessary store to mapping
         if ":" not in resultname:
+            logger.info( "storing {resultname=}" )
             if self.resolverobject.resolve_property_name_to_uri is not None:
                 result1 = self.resolverobject.resolve_property_name_to_uri(resultname)
                 if result1 is None:
@@ -647,6 +655,7 @@ def identifier(self, s):
             # a prefixed name is assumed to be usable directly (the prefix has been added to prefixes)
             prefix = resultname.split( ":",1 )[0]
             self.prefixes[rdfxml.RDF_DEFAULT_PREFIX[prefix]] = prefix
+            logger.info( f"Added prefix {prefix=}" )
             result = resultname
         logger.info( f"identifier1 {result=}" )
         return result

elmclient/_rm.py

@@ -516,6 +516,7 @@ def load_configs(self, cacheable=True):
                     thisconfu = confmember.get("{%s}resource" % rdfxml.RDF_DEFAULT_PREFIX["rdf"])
                     self._confs_to_load.append(thisconfu)
 #                    print( f"   Adding {thisconfu=}" )
+#                    print( f"   Now {self._confs_to_load=}" )
 
             # maybe it's got configuration(s)
             confmembers_x = list( set( rdfxml.xml_find_elements(configs_xml, './/oslc_config:Configuration') + rdfxml.xml_find_elements(configs_xml, './/oslc_config:Stream') + rdfxml.xml_find_elements(configs_xml, './/oslc_config:Baseline') + rdfxml.xml_find_elements(configs_xml, './/oslc_config:ChangeSet') ) )
@@ -541,7 +542,7 @@ def load_configs(self, cacheable=True):
                 created = rdfxml.xmlrdf_get_resource_uri(confmember_x, './dcterms:created', exceptionifnotfound=True)
 #                print( f"{conftitle=}" )
 #                print( f"{created=}" )
-                if confmember_x.tag == '{http://open-services.net/ns/config#}Changeset':
+                if confmember_x.tag == '{http://open-services.net/ns/config#}ChangeSet':
                     conftype = "ChangeSet"
                     ischangeset=True
                 elif confmember_x.tag == '{http://open-services.net/ns/config#}Baseline':
@@ -624,6 +625,7 @@ def load_configs(self, cacheable=True):
 
     def get_local_config(self, name_or_uri, global_config_uri=None):
         logger.info( f"GLC {self=} {name_or_uri=}" )
+#        print( f"GLC {self=} {name_or_uri=} {global_config_uri=}" )
         if global_config_uri:
             # gc and local config both specified - try to avoid loading all the local configs by using the gc tree to locate the local config
             gc_contribs = self.get_gc_contributions(global_config_uri)
@@ -650,12 +652,15 @@ def get_local_config(self, name_or_uri, global_config_uri=None):
                 filter="ChangeSet"
                 name_or_uri = name_or_uri[2:]
             for cu, cd in self._configurations.items():
+#                print( f"{cu=} {cd=}" )
                 if filter and cd['conftype'] != filter:
                     continue
                 if cu == name_or_uri or cd['name'] == name_or_uri:
                     if result:
                         raise Exception( f"Config {name_or_uri} isn't unique - you could try prefixing it with S: for stream, B: for baseline, or C: for changeset")
                     result = cu
+#        print( f"GLC {result} {self=} {name_or_uri=}" )
+                    
         return result
 
     def list_configs( self ):

elmclient/_typesystem.py

@@ -293,7 +293,7 @@ def get_enum_id( self, enum_name, property_uri ):
         for enum_uri in self.properties[property_uri]['enums']:
             if self.enums[enum_uri]['name']==enum_name:
                 result = self.enums[enum_uri]['id'] or enum_uri
-                result = enum_uri
+#                result = enum_uri # this makes ccm queries for e.g. rc:cm:type=Defect not work - ccm doens't like getting a URI - # unfortunately I can't remember why I added this line :-(
                 break
         logger.info( f"get_enum_id {enum_name=} {property_uri=} {result=}" )
         return result

elmclient/examples/reqif_io.py

@@ -36,6 +36,9 @@
 import elmclient._app as _app
 import elmclient.utils as utils
 
+# disable caching completely
+cachecontrol=2
+
 ############################################################################
 
 def reqif_main():
@@ -71,7 +74,7 @@ def reqif_main():
     parser.add_argument("-P", "--password", default=PASSWORD, help=f"User password default '{PASSWORD}' - can be set using env variable OUERY_PASSWORD - set to PROMPT to be prompted at runtime")
     parser.add_argument('-T', '--certs', action="store_true", help="Verify SSL certificates")
     parser.add_argument("-U", "--username", default=USER, help="User id - can be set using environment variable QUERY_USER")
-    parser.add_argument('-W', '--cachecontrol', action='count', default=0, help="Used once -W erases cache then continues with caching enabled. Used twice -WW wipes cache and disables caching. Otherwise caching is continued from previous run(s).")
+#    parser.add_argument('-W', '--cachecontrol', action='count', default=0, help="Used once -W erases cache then continues with caching enabled. Used twice -WW wipes cache and disables caching. Otherwise caching is continued from previous run(s).")
     parser.add_argument('-Z', '--proxyport', default=8888, type=int, help='Port for proxy default is 8888 - used if found to be active - set to 0 to disable')
 
     # saved credentials
@@ -206,7 +209,7 @@ def reqif_main():
         approots['jts']='jts'
 
     # create our "server"
-    theserver = server.JazzTeamServer(args.jazzurl, args.username, args.password, verifysslcerts=args.certs,appstring=f"jts:{approots['jts']}",cachingcontrol=args.cachecontrol)
+    theserver = server.JazzTeamServer(args.jazzurl, args.username, args.password, verifysslcerts=args.certs,appstring=f"jts:{approots['jts']}",cachingcontrol=cachecontrol)
 
     # create all our apps
     for appdom,approot in approots.items():
@@ -440,7 +443,7 @@ def getmatchingdefs(alldefs,definitionnames):
 
             # execute post content
             logger.info('Uploading package {ifile}...')
-            response = queryon.execute_post_content(pkg_factory_u, data=body, headers={'Content-Type': content_type, 'userMimeType': 'application/zip', 'filename':os.path.basename(ifile),'Accept': '*/*','X-Requested-With': None,'Origin': 'https://jazz.ibm.com:9443', 'OSLC-Core-Version': None}, intent="Initiate reqif export " )
+            response = queryon.execute_post_content(pkg_factory_u, data=body, headers={'Content-Type': content_type, 'userMimeType': 'application/zip', 'filename':os.path.basename(ifile),'Accept': '*/*','X-Requested-With': None,'Origin': 'https://jazz.ibm.com:9443', 'OSLC-Core-Version': None}, intent="Initiate reqif import - upload the reqif file" )
 
             print( f"Triggering import for {os.path.basename(ifile)}" )
 
@@ -465,7 +468,7 @@ def getmatchingdefs(alldefs,definitionnames):
 </rdf:RDF>"""
 
             logger.debug( f"{content=}" )
-            response = queryon.execute_post_rdf_xml( import_factory_u, data=content, cacheable=False, headers={'net.jazz.jfs.owning-context': queryon.project_uri, 'OSLC-Core-Version': None}, intent="Initiate reqif import" )
+            response = queryon.execute_post_rdf_xml( import_factory_u, data=content, cacheable=False, headers={'net.jazz.jfs.owning-context': queryon.project_uri, 'OSLC-Core-Version': None}, intent="Initiate reqif import of the uploaded file" )
             logger.debug( f" {response.status_code=} {response=}" )
             location = response.headers.get('Location')
             if response.status_code == 202 and location is not None:

elmclient/examples/trsreader.py

@@ -6,13 +6,17 @@
 #
 # Read and process TRS base and changelog pages
 #
-# NOTE: UNFINISHED WORK IN PROGRESS!
+#################################################
+#################################################
+# NOTE: VERY VERY UNFINISHED WORK IN PROGRESS!
+#################################################
+#################################################
 #
 # NOTE this requires that your user has the license trs_consumer assigned, which allows access to the TRS.
 #  __DON'T__ reassign a license from lqe_user to yourself because then LQE will stop working!
 #
 # If you have the trs license you can access all the content in the TRS,
-# BUT note that the TRS basically contains URLs pointing at e.g. DOORS Next resources, so there are no text of artifacts, etc., but you can say that a resource is in a certain configuration
+# BUT note that the TRS basically contains URLs pointing at e.g. DOORS Next resources, so there is no text of artifacts, etc., but you can say that a resource is in a certain configuration
 # AND note that retrieving the data from DOORS Next is constrained by the access right of your user
 #
 

elmclient/httpops.py

@@ -722,17 +722,23 @@ def _jsa_login(self, auth_url):
             auth_url_response = self._session.get(str(auth_url))  # Load up them cookies!
             self.log_redirection_history( auth_url_response, intent="JAS Authorize step 3",donotlogbody=True )
             if auth_url_response.status_code == 200:
-                login_url = auth_url_response.url  # Take the redirected URL and login action URL
-                logger.debug( f"1 {login_url=}" )
-                if auth_url != login_url:
-                    content_text = to_text(auth_url_response)
-                    #                 print content
-                    auth_form_parser = _FormParser()
-                    auth_form_parser.feed(content_text)
-                    if auth_form_parser.action:
-                        login_url = urllib.parse.urljoin(login_url, auth_form_parser.action)
-                        logger.debug( f"2 {login_url=}" )
-                self._authorize(login_url)
+                if True:
+                    # use basic auth - 3iii in https://jazz.net/wiki/bin/view/Main/NativeClientAuthentication
+                    username, password = self.get_user_password(auth_url)
+                    auth_url_response = self._session.get( str(auth_url), auth=(username, password) )  # Load up them cookies!
+                else:
+                    # use form auth
+                    login_url = auth_url_response.url  # Take the redirected URL and login action URL
+                    logger.debug( f"1 {login_url=}" )
+                    if auth_url != login_url:
+                        content_text = to_text(auth_url_response)
+                        #                 print content
+                        auth_form_parser = _FormParser()
+                        auth_form_parser.feed(content_text)
+                        if auth_form_parser.action:
+                            login_url = urllib.parse.urljoin(login_url, auth_form_parser.action)
+                            logger.debug( f"2 {login_url=}" )
+                    self._authorize(login_url)
         else:
             logger.error('''Something about JSA OIDC login has changed since this script was written. I can no longer determine where to authorize myself.''')
             raise Exception("Authorize not possible (1)")