import io import os import shutil import unittest from contextlib import contextmanager from glob import glob from lxml import etree as ET from odml.terminology import REPOSITORY_BASE from odml.tools.converters import VersionConverter from .util import ODML_CACHE_DIR as CACHE_DIR, create_test_dir, TEST_RESOURCES_DIR as RES_DIR try: unicode = unicode except NameError: unicode = str class TestVersionConverter(unittest.TestCase): def setUp(self): self.base_path = RES_DIR self.VC = VersionConverter self.doc = """ 2008-07-07
0degint 45degint Orientations some sec type sec_name
some sec type sec_name prop_name prop_name
Author
""" self.tmp_dir = None def tearDown(self): """ Cleanup any created temporary files. """ if self.tmp_dir and os.path.exists(self.tmp_dir): shutil.rmtree(self.tmp_dir) find_us = os.path.join(CACHE_DIR, "*local_repository_file_v1*") for file_path in glob(find_us): os.remove(file_path) @contextmanager def assertNotRaises(self, exc_type): try: yield None except exc_type: raise self.failureException('{} raised'.format(exc_type.__name__)) def test_replace_same_name_entites(self): root = ET.fromstring(self.doc) sec_names = [] sec_elems = [] for sec in root.iter("section"): sec_names.append(sec.find("name").text) sec_elems.append(sec) self.assertEqual(sec_names[0], "sec_name") self.assertEqual(sec_names[0], sec_names[1]) props_names = [] for prop in sec_elems[1].iter("property"): props_names.append(prop.find("name").text) self.assertEqual(props_names[0], "prop_name") self.assertEqual(props_names[0], props_names[1]) tree = ET.ElementTree(root) tree = self.VC._replace_same_name_entities(tree) root = tree.getroot() sec_names = [] sec_elems = [] for sec in root.iter("section"): sec_names.append(sec.find("name").text) sec_elems.append(sec) self.assertEqual(sec_names[0], "sec_name") self.assertEqual(sec_names[1], "sec_name-2") props_names = [] for prop in sec_elems[1].iter("property"): props_names.append(prop.find("name").text) self.assertEqual(props_names[0], "prop_name") self.assertEqual(props_names[1], "prop_name-2") def test_convert_odml_file(self): with self.assertRaises(Exception) as exc: self.VC("/not_valid_path").convert() self.assertIn("Cannot parse provided file", str(exc.exception)) root = ET.fromstring(self.doc) prop = root.find("section").find("property") val_elems = [] for val in prop.iter("value"): val_elems.append(val) self.assertEqual(val_elems[0].find("unit").text, "deg") self.assertEqual(val_elems[0].find("type").text, "int") self.assertEqual(val_elems[0].find("uncertainty").text, None) self.assertEqual(prop.find("unit"), None) self.assertEqual(prop.find("type"), None) file = io.StringIO(unicode(self.doc)) converter = self.VC(file) tree = converter._convert(converter._parse_xml()) root = tree.getroot() prop = root.find("section").find("property") val_elems = [] for val in prop.iter("value"): val_elems.append(val) self.assertEqual(len(val_elems), 1) self.assertEqual(val_elems[0].find("unit"), None) self.assertEqual(val_elems[0].find("type"), None) self.assertEqual(val_elems[0].find("uncertainty"), None) self.assertEqual(val_elems[0].text, "[0,45]") self.assertEqual(prop.find("unit").text, "deg") self.assertEqual(len(prop.findall("unit")), 1) self.assertEqual(prop.find("type").text, "int") self.assertEqual(len(prop.findall("type")), 1) self.assertEqual(prop.find("uncertainty").text, None) def test_convert_odml_file_document(self): """Test proper conversion of the odml.Document entity from odml model version 1 to version 1.1. The test checks for the proper conversion of all valid Document tags and exclusion of non-Document tags. """ repo_file = os.path.join(RES_DIR, "local_repository_file_v1.1.xml") local_url = "file://%s" % repo_file repo_old_file = os.path.join(RES_DIR, "local_repository_file_v1.0.xml") local_old_url = "file://%s" % repo_old_file doc = """ Document author 1 2017-10-18 %s
Document section
Invalid Document tag Invalid Document property Invalid Document value
""" % local_url invalid_repo_doc = """ Unresolvable
Document section
""" old_repo_doc = """ %s
Document section
""" % local_old_url file = io.StringIO(unicode(doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) root = conv_doc.getroot() # Test export of Document tags, repository is excluded self.assertEqual(len(root.findall("author")), 1) self.assertEqual(len(root.findall("date")), 1) self.assertEqual(len(root.findall("repository")), 1) self.assertEqual(len(root.findall("section")), 1) # Test absence of non-Document tags self.assertEqual(len(root.findall("invalid")), 0) self.assertEqual(len(root.findall("property")), 0) self.assertEqual(len(root.findall("value")), 0) # Test warning message on non-importable repository file = io.StringIO(unicode(invalid_repo_doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) root = conv_doc.getroot() self.assertEqual(root.findall("repository")[0].text, "Unresolvable") self.assertIn("not odML v1.1 compatible", converter.conversion_log[0]) # Test warning message on old repository file = io.StringIO(unicode(old_repo_doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) root = conv_doc.getroot() self.assertEqual(root.findall("repository")[0].text, local_old_url) self.assertIn("not odML v1.1 compatible", converter.conversion_log[0]) def test_convert_odml_file_section(self): """Test proper conversion of the odml.Section entity from odml model version 1 to version 1.1. The test checks for the proper conversion of all valid Section tags and exclusion of non-Section tags. """ repo_file = os.path.join(RES_DIR, "local_repository_file_v1.1.xml") local_url = "file://%s" % repo_file repo_old_file = os.path.join(RES_DIR, "local_repository_file_v1.0.xml") local_old_url = "file://%s" % repo_old_file doc = """
Section name Section type Section definition Section reference Section link %s Section include Section Property 1 Section Property 2
SubSection name SubSection type SubSection definition SubSection reference SubSection link %s SubSection include SubSection Property
Unsupported Section tags test Invalid tag Invalid Value tag Unsupported mapping tag
""" % (local_url, local_url) file = io.StringIO(unicode(doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) root = conv_doc.getroot() root_id = root.findall("id") self.assertEqual(len(root_id), 1) sec = root.findall("section") self.assertEqual(len(sec), 2) # Test valid section tags. self.assertEqual(len(sec[0]), 11) self.assertEqual(sec[0].find("name").text, "Section name") self.assertEqual(sec[0].find("type").text, "Section type") self.assertEqual(sec[0].find("definition").text, "Section definition") self.assertEqual(sec[0].find("reference").text, "Section reference") self.assertEqual(sec[0].find("link").text, "Section link") self.assertEqual(sec[0].find("repository").text, local_url) self.assertEqual(sec[0].find("include").text, "Section include") self.assertEqual(len(sec[0].findall("property")), 2) self.assertEqual(len(sec[0].findall("section")), 1) self.assertEqual(len(sec[0].findall("id")), 1) # Test valid subsection tags. subsec = sec[0].find("section") self.assertEqual(len(subsec), 9) self.assertEqual(subsec.find("name").text, "SubSection name") self.assertEqual(subsec.find("type").text, "SubSection type") self.assertEqual(subsec.find("definition").text, "SubSection definition") self.assertEqual(subsec.find("reference").text, "SubSection reference") self.assertEqual(subsec.find("link").text, "SubSection link") self.assertEqual(subsec.find("repository").text, local_url) self.assertEqual(subsec.find("include").text, "SubSection include") self.assertEqual(len(subsec.findall("property")), 1) self.assertEqual(len(subsec.findall("id")), 1) # Test absence of non-Section tags self.assertEqual(len(sec[1]), 2) self.assertEqual(len(sec[1].findall("name")), 1) self.assertEqual(len(sec[1].findall("id")), 1) # Test presence of v1.0 repository tag and warning log entry doc = """
Unsupported Section include test %s
""" % local_old_url file = io.StringIO(unicode(doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) sec = conv_doc.getroot().findall("section") self.assertEqual(sec[0].find("repository").text, local_old_url) self.assertIn("not odML v1.1 compatible", converter.conversion_log[0]) # Test presence of v1.0 include tag and warning log entry doc = """
Unsupported Section include test %s
""" % local_old_url file = io.StringIO(unicode(doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) sec = conv_doc.getroot().findall("section") self.assertEqual(sec[0].find("include").text, local_old_url) self.assertIn("not odML v1.1 compatible", converter.conversion_log[0]) def test_convert_odml_file_property(self): """Test proper conversion of the odml.Property entity from odml model version 1 to version 1.1. The test checks for the proper conversion of all valid Property tags and exclusion of non-Property tags. """ doc = """
Valid Property tags test Property name Property type Property definition Property dependency Property dependency value
Unsupported Property tags test Invalid Property Invalid tag
Invalid Section
Property with no name
""" file = io.StringIO(unicode(doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) root = conv_doc.getroot() sec = root.findall("section") # Test valid Property tags self.assertEqual(sec[0].find("name").text, "Valid Property tags test") self.assertEqual(len(sec[0].findall("property")), 1) prop = sec[0].find("property") self.assertEqual(len(prop), 6) self.assertEqual(prop.find("name").text, "Property name") self.assertEqual(prop.find("type").text, "Property type") self.assertEqual(prop.find("definition").text, "Property definition") self.assertEqual(prop.find("dependency").text, "Property dependency") self.assertEqual(prop.find("dependencyvalue").text, "Property dependency value") self.assertEqual(len(prop.findall("id")), 1) # Test non-import of Property w/o name self.assertEqual(len(sec[1].findall("property")), 1) # Test absence of non-Property tags prop = sec[1].find("property") self.assertEqual(len(prop), 2) self.assertEqual(len(prop.findall("name")), 1) self.assertEqual(len(prop.findall("id")), 1) def test_convert_odml_file_value(self): """Test proper conversion of the odml.Value entity from odml model version 1 to version 1.1. The test checks for the proper conversion of all valid Value tags and exclusion of non-Value tags. """ doc = """
Values export test Single value export 1 Multiple values export 1 2 3 Empty value export Supported Value tags export 0.1 float 0.05 mV raw.txt Value reference Supported Multiple Value tags export 0.1 mV raw.txt Value reference 0.2 float 0.05 mV raw.txt Value reference 3 int 0.06 kV raw2.txt Value reference 2 Unsupported Value tags export Invalid Value tag Encoder Checksum Unsupported binary value type replace 0 binary Unsupported binary value dtype replace 1 binary Single, string, value, with, many, commata.string testSingleString Astring Bstring Cstring testStringList Single string value with wrapping whitespace string testStringWhiteSpace Multiple Strings string with wrapping string Whitespace string testStringListWhiteSpace 1 int 2 int 3 int testIntListWhiteSpace Single value with UUID 1 aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa Single value with none-UUID ID 1 Single value with empty ID
""" file = io.StringIO(unicode(doc)) converter = self.VC(file) conv_doc = converter._convert(converter._parse_xml()) root = conv_doc.getroot() root_id = root.findall("id") self.assertEqual(len(root_id), 1) sec = root.find("section") self.assertEqual(len(sec), 18) self.assertEqual(len(sec.findall("id")), 1) # Test single value export prop = sec.findall("property")[0] self.assertEqual(len(prop), 3) self.assertEqual(prop.find("value").text, "1") self.assertEqual(len(prop.findall("id")), 1) # Test multiple value export prop = sec.findall("property")[1] self.assertEqual(len(prop), 3) self.assertEqual(prop.find("value").text, "[1,2,3]") self.assertEqual(len(prop.findall("id")), 1) # Test empty value export prop = sec.findall("property")[2] self.assertEqual(len(prop), 2) self.assertEqual(prop.find("name").text, "Empty value export") self.assertEqual(len(prop.findall("id")), 1) # Test valid Value tags prop = sec.findall("property")[3] self.assertEqual(len(prop), 8) self.assertEqual(prop.find("value").text, "0.1") self.assertEqual(prop.find("type").text, "float") self.assertEqual(prop.find("uncertainty").text, "0.05") self.assertEqual(prop.find("unit").text, "mV") self.assertEqual(prop.find("value_origin").text, "raw.txt") self.assertEqual(prop.find("reference").text, "Value reference") self.assertEqual(len(prop.findall("filename")), 0) self.assertEqual(len(prop.findall("id")), 1) # Test valid multiple Value tag export prop = sec.findall("property")[4] self.assertEqual(len(prop), 8) self.assertEqual(prop.find("value").text, "[0.1,0.2,3]") self.assertEqual(prop.find("type").text, "float") self.assertEqual(prop.find("uncertainty").text, "0.05") self.assertEqual(prop.find("unit").text, "mV") self.assertEqual(prop.find("value_origin").text, "raw.txt") self.assertEqual(prop.find("reference").text, "Value reference") self.assertEqual(len(prop.findall("id")), 1) # Test non-export of invalid Value tags prop = sec.findall("property")[5] self.assertEqual(len(prop), 2) self.assertEqual(len(prop.findall("name")), 1) self.assertEqual(len(prop.findall("id")), 1) # Test dtype 'binary' replacement prop = sec.findall("property")[6] self.assertEqual(prop.find("name").text, "Unsupported binary value type replace") self.assertEqual(prop.find("type").text, "text") self.assertEqual(len(prop.findall("id")), 1) prop = sec.findall("property")[7] self.assertEqual(prop.find("name").text, "Unsupported binary value dtype replace") self.assertEqual(prop.find("type").text, "text") self.assertEqual(len(prop.findall("id")), 1) # Test single string value with commata prop = sec.findall("property")[8] self.assertEqual(prop.find("name").text, "testSingleString") self.assertEqual(prop.find("value").text, "Single, string, value, with, many, commata.") self.assertEqual(len(prop.findall("id")), 1) # Test string list import prop = sec.findall("property")[9] self.assertEqual(prop.find("name").text, "testStringList") self.assertEqual(prop.find("value").text, "[A,B,C]") self.assertEqual(len(prop.findall("id")), 1) # Test single string values wrapping whitespace removal prop = sec.findall("property")[10] self.assertEqual(prop.find("name").text, "testStringWhiteSpace") self.assertEqual(prop.find("value").text, "Single string value with wrapping whitespace") self.assertEqual(len(prop.findall("id")), 1) # Test multiple string values with wrapping whitespace removal prop = sec.findall("property")[11] self.assertEqual(prop.find("name").text, "testStringListWhiteSpace") self.assertEqual(prop.find("value").text, "[Multiple Strings,with wrapping,Whitespace]") self.assertEqual(len(prop.findall("id")), 1) # Test multiple int values with wrapping whitespaces prop = sec.findall("property")[12] self.assertEqual(prop.find("name").text, "testIntListWhiteSpace") self.assertEqual(prop.find("type").text, "int") self.assertEqual(prop.find("value").text, "[1,2,3]") self.assertEqual(len(prop.findall("id")), 1) # Test single value export prop = sec.findall("property")[13] self.assertEqual(len(prop), 3) self.assertEqual(prop.find("value").text, "1") self.assertEqual(len(prop.findall("id")), 1) self.assertEqual(prop.find("id").text, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") # Test single value export prop = sec.findall("property")[14] self.assertEqual(len(prop), 2) self.assertEqual(len(prop.findall("id")), 1) self.assertNotEqual(prop.find("id").text, "1") # Test single value export prop = sec.findall("property")[15] self.assertEqual(len(prop), 2) self.assertEqual(len(prop.findall("id")), 1) self.assertNotEqual(prop.find("id").text, "1") def test_parse_dict_document(self): # Test appending tags; not appending empty sections doc_dict = {'Document': {'author': 'HPL', 'sections': []}} root = self.VC("")._parse_dict_document(doc_dict).getroot() self.assertEqual(len(root.getchildren()), 1) self.assertIsNotNone(root.find("author")) # Test appending multiple sections doc_dict = {'Document': {'author': 'HPL', 'sections': [{'name': 'sec_one'}, {'name': 'sec_two'}]}} root = self.VC("")._parse_dict_document(doc_dict).getroot() self.assertEqual(len(root.getchildren()), 3) self.assertEqual(len(root.findall("section")), 2) def test_parse_dict_sections(self): # Test appending tags; not appending empty subsections or properties root = ET.Element("root") sec_dict = [{'name': 'sec_one', 'sections': [], 'properties': []}] self.assertEqual(len(root.getchildren()), 0) self.VC("")._parse_dict_sections(root, sec_dict) self.assertEqual(len(root.getchildren()), 1) self.assertIsNotNone(root.find("section").find("name")) # Test appending multiple sections root = ET.Element("root") sec_dict = [{'name': 'sec_one'}, {'name': 'sec_two'}, {'name': 'sec_three'}] self.assertEqual(len(root.getchildren()), 0) self.VC("")._parse_dict_sections(root, sec_dict) self.assertEqual(len(root.getchildren()), 3) # Test appending subsections root = ET.Element("root") sec_dict = [{'name': 'sec_one', 'sections': [{'name': 'sub_one'}, {'name': 'sub_two'}]}] self.assertEqual(len(root.getchildren()), 0) self.VC("")._parse_dict_sections(root, sec_dict) sec = root.find("section") self.assertEqual(len(sec.getchildren()), 3) self.assertEqual(len(sec.findall("section")), 2) # Test appending properties root = ET.Element("root") sec_dict = [{'name': 'sec_one', 'properties': [{'name': 'prop_one'}, {'name': 'prop_two'}]}] self.assertEqual(len(root.getchildren()), 0) self.VC("")._parse_dict_sections(root, sec_dict) sec = root.find("section") self.assertEqual(len(sec.getchildren()), 3) self.assertEqual(len(sec.findall("property")), 2) def test_parse_dict_properties(self): # Test appending tags and moving values root = ET.Element("root") prop_dict = [{'name': 'prop_one', 'values': [{'unit': 'none'}, {'value': '1'}]}] self.assertEqual(len(root.getchildren()), 0) self.VC("")._parse_dict_properties(root, prop_dict) self.assertEqual(len(root.getchildren()), 1) self.assertIsNotNone(root.find("property")) prop = root.find("property") self.assertEqual(len(prop.getchildren()), 3) self.assertIsNotNone(prop.find("name")) self.assertEqual(len(prop.findall("value")), 2) # Test multiple entries root = ET.Element("root") prop_dict = [{'name': 'prop_one'}, {'name': 'prop_two'}] self.assertEqual(len(root.getchildren()), 0) self.VC("")._parse_dict_properties(root, prop_dict) self.assertEqual(len(root.getchildren()), 2) def test_parse_dict_values(self): root = ET.Element("root") val_dict = [{'unit': 'arbitrary', 'value': "['one', 'two']"}, {'unit': 'mV', 'value': '1'}] self.VC("")._parse_dict_values(root, val_dict) self.assertEqual(len(root.getchildren()), 2) for val in root.iterchildren(): self.assertEqual(val.tag, "value") self.assertEqual(len(val.getchildren()), 1) self.assertIsNotNone(val.find("unit")) self.assertIsNotNone(val.text) def test_handle_repository(self): repo = ET.Element("repository") # Test working and valid repository link repo.text = '/'.join([REPOSITORY_BASE, 'v1.1', 'analysis', 'analysis.xml']) converter = self.VC("") self.assertIsNone(converter._handle_repository(repo)) self.assertEqual(converter.conversion_log, []) # Test replaced working repository link repo.text = '/'.join([REPOSITORY_BASE, 'v1.0', 'analysis', 'analysis.xml']) self.assertIsNone(converter._handle_repository(repo)) self.assertEqual(repo.text, '/'.join([REPOSITORY_BASE, 'v1.1', 'analysis', 'analysis.xml'])) self.assertEqual(len(converter.conversion_log), 1) self.assertTrue("[Info]" in converter.conversion_log[0]) # Test invalid repository link invalid = "I am leading nowhere" repo.text = invalid self.assertIsNone(converter._handle_repository(repo)) self.assertEqual(len(converter.conversion_log), 2) self.assertTrue("[Warning]" in converter.conversion_log[1]) self.assertEqual(repo.text, invalid) def test_handle_include(self): repo = ET.Element("include") # Test working and valid repository link repo.text = '/'.join([REPOSITORY_BASE, 'v1.1', 'analysis', 'analysis.xml']) converter = self.VC("") self.assertIsNone(converter._handle_include(repo)) self.assertEqual(converter.conversion_log, []) # Test replaced working repository link repo.text = '/'.join([REPOSITORY_BASE, 'v1.0', 'analysis', 'analysis.xml']) self.assertIsNone(converter._handle_include(repo)) self.assertEqual(repo.text, '/'.join([REPOSITORY_BASE, 'v1.1', 'analysis', 'analysis.xml'])) self.assertEqual(len(converter.conversion_log), 1) self.assertTrue("[Info]" in converter.conversion_log[0]) # Test invalid repository link invalid = "I am leading nowhere" repo.text = invalid self.assertIsNone(converter._handle_include(repo)) self.assertEqual(len(converter.conversion_log), 2) self.assertTrue("[Warning]" in converter.conversion_log[1]) self.assertEqual(repo.text, invalid) def test_convert_xml_file(self): # Test minimal reading from an xml file. basefile = os.path.join(self.base_path, "version_conversion.xml") root = self.VC(basefile)._parse_xml().getroot() self.assertIsNotNone(root.find("section")) sec = root.find("section") self.assertIsNotNone(sec.find("name")) self.assertIsNotNone(sec.find("type")) self.assertIsNotNone(sec.find("section").find("name")) self.assertIsNotNone(sec.find("property")) prop = sec.find("property") self.assertIsNotNone(prop.find("name")) self.assertIsNotNone(prop.find("value")) def test_convert_yaml_file(self): # Test minimal reading from a yaml file. basefile = os.path.join(self.base_path, "version_conversion.yaml") root = self.VC(basefile)._parse_yaml().getroot() self.assertIsNotNone(root.find("section")) sec = root.find("section") self.assertIsNotNone(sec.find("name")) self.assertIsNotNone(sec.find("type")) self.assertIsNotNone(sec.find("section")) self.assertIsNotNone(sec.find("property")) prop = sec.find("property") self.assertIsNotNone(prop.find("name")) self.assertIsNotNone(prop.find("value")) def test_convert_json_file(self): # Test minimal reading from a json file. basefile = os.path.join(self.base_path, "version_conversion.json") root = self.VC(basefile)._parse_json().getroot() self.assertIsNotNone(root.find("section")) sec = root.find("section") self.assertIsNotNone(sec.find("name")) self.assertIsNotNone(sec.find("type")) self.assertIsNotNone(sec.find("section")) self.assertIsNotNone(sec.find("property")) prop = sec.find("property") self.assertIsNotNone(prop.find("name")) self.assertIsNotNone(prop.find("value")) def test_write_to_file(self): infile = os.path.join(self.base_path, "version_conversion.xml") self.tmp_dir = create_test_dir(__file__) # Test write to named file outfile = os.path.join(self.tmp_dir, "test.odml") self.VC(infile).write_to_file(outfile) self.assertTrue(os.path.exists(outfile)) # Test file extension append write to named file w/o file extension outfile = os.path.join(self.tmp_dir, "test") self.VC(infile).write_to_file(outfile) self.assertFalse(os.path.exists(outfile)) self.assertTrue(os.path.exists("%s.xml" % outfile))