glossary test: added testcase for to_str() without description

Introduces several changes with the main objective of enabling OpenAI's streaming API in the chatmastermind application. This allows for the retrieval of AI responses gradually as a stream, which can significantly improve the user experience in interactions that involve large result sets.

* Added tiktoken import in 'openai.py' and modifications to the OpenAI class to support streaming. This includes the addition of a new class OpenAIAnswer to handle streaming API responses.
* Modified request function in the OpenAI class: the stream=True flag is added to the openai.ChatCompletion.create method to enable streaming API.
* Modified 'question.py' to print the answer parts as they are streamed.
* Replaced the Answer class's string data type with a generator which supports str and Generator[str, None, None] data types. Modifications are made to the Answer class methods to handle both data types accordingly.
* Updated the tests in 'test_ais_openai.py' and 'test_message.py' to reflect and validate these changes.

chatmastermind/commands/glossary.py

@@ -1,6 +1,7 @@
 import sys
 import argparse
 from pathlib import Path
+from pydoc import pager
 from ..configuration import Config
 from ..glossary import Glossary
 
@@ -9,6 +10,10 @@ class GlossaryCmdError(Exception):
     pass
 
 
+def print_paged(text: str) -> None:
+    pager(text)
+
+
 def get_glossary_file_path(name: str, config: Config) -> Path:
     """
     Get the complete filename for a glossary with the given path.
@@ -24,9 +29,27 @@ def list_glossaries(args: argparse.Namespace, config: Config) -> None:
     """
     if not config.glossaries:
         raise GlossaryCmdError("Glossaries directory missing in the configuration file")
-    glossaries = Path(config.glossaries).glob(f'*.{Glossary.file_suffix}')
-    for glo in glossaries:
-        print(Glossary.from_file(glo).to_str(args.entries))
+    glossaries = Path(config.glossaries).glob(f'*{Glossary.file_suffix}')
+    for glo in sorted(glossaries):
+        print(Glossary.from_file(glo).to_str())
+
+
+def print_glossary(args: argparse.Namespace, config: Config) -> None:
+    """
+    Print an existing glossary.
+    """
+    # sanity checks
+    if args.name is None:
+        raise GlossaryCmdError("Missing glossary name")
+    if config.glossaries is None and args.file is None:
+        raise GlossaryCmdError("Glossaries directory missing in the configuration file")
+    # create file path or use the given one
+    glo_file = Path(args.file) if args.file else get_glossary_file_path(args.name, config)
+    if not glo_file.exists():
+        raise GlossaryCmdError(f"Glossary '{glo_file}' does not exist")
+    # read glossary
+    glo = Glossary.from_file(glo_file)
+    print_paged(glo.to_str(with_entries=True))
 
 
 def create_glossary(args: argparse.Namespace, config: Config) -> None:
@@ -34,24 +57,19 @@ def create_glossary(args: argparse.Namespace, config: Config) -> None:
     Create a new glossary and write it either to the glossaries directory
     or the given file.
     """
-    # we need to know where the glossary should be stored
-    if config.glossaries is None and args.file is None:
-        raise GlossaryCmdError("Glossaries directory missing in the configuration file")
     # sanity checks
     if args.name is None:
-        print("Error: please specify the glossary name.")
-        sys.exit(1)
+        raise GlossaryCmdError("Missing glossary name")
     if args.source_lang is None:
-        print("Error: please specify the source language.")
-        sys.exit(1)
+        raise GlossaryCmdError("Missing source language")
     if args.target_lang is None:
-        print("Error: please specify the target language.")
-        sys.exit(1)
-    # create file or use the given one
+        raise GlossaryCmdError("Missing target language")
+    if config.glossaries is None and args.file is None:
+        raise GlossaryCmdError("Glossaries directory missing in the configuration file")
+    # create file path or use the given one
     glo_file = Path(args.file) if args.file else get_glossary_file_path(args.name, config)
     if glo_file.exists():
-        print(f"Error: glossary '{glo_file}' already exists!")
-        sys.exit(1)
+        raise GlossaryCmdError(f"Glossary '{glo_file}' already exists")
     glo = Glossary(name=args.name,
                    source_lang=args.source_lang,
                    target_lang=args.target_lang,
@@ -65,5 +83,11 @@ def glossary_cmd(args: argparse.Namespace, config: Config) -> None:
     """
     Handler for the 'glossary' command.
     """
-    if args.create:
-        create_glossary(args, config)
+    try:
+        if args.create:
+            create_glossary(args, config)
+        elif args.list:
+            list_glossaries(args, config)
+    except GlossaryCmdError as err:
+        print(f"Error: {err}")
+        sys.exit(1)

chatmastermind/configuration.py

@@ -150,6 +150,8 @@ class Config:
 
     @classmethod
     def from_file(cls: Type[ConfigInst], path: str) -> ConfigInst:
+        if not Path(path).exists():
+            raise ConfigError(f"Configuration file '{path}' not found. Use 'cmm config --create' to create one.")
         with open(path, 'r') as f:
             source = yaml.load(f, Loader=yaml.FullLoader)
         return cls.from_dict(source)

chatmastermind/glossary.py

@@ -95,7 +95,7 @@ class Glossary:
         with tempfile.NamedTemporaryFile(dir=self.file_path.parent, prefix=self.file_path.name, mode="w", delete=False) as temp_fd:
             temp_file_path = Path(temp_fd.name)
             data = {'Name': self.name,
-                    'Description': self.desc,
+                    'Description': str(self.desc),
                     'ID': str(self.ID),
                     'SourceLang': self.source_lang,
                     'TargetLang': self.target_lang,
@@ -153,7 +153,7 @@ class Glossary:
         """
         output: list[str] = []
         output.append(f'{self.name} (ID: {self.ID}):')
-        if self.desc:
+        if self.desc and self.desc != 'None':
             output.append('- ' + self.desc)
         output.append(f'- Languages: {self.source_lang} -> {self.target_lang}')
         if with_entries:

chatmastermind/main.py

@@ -8,7 +8,7 @@ import argcomplete
 import argparse
 from pathlib import Path
 from typing import Any
-from .configuration import Config, default_config_file
+from .configuration import Config, default_config_file, ConfigError
 from .message import Message
 from .commands.question import question_cmd
 from .commands.tags import tags_cmd
@@ -55,7 +55,7 @@ def create_parser() -> argparse.ArgumentParser:
     ai_parser = argparse.ArgumentParser(add_help=False)
     ai_parser.add_argument('-A', '--AI', help='AI ID to use', metavar='AI_ID')
     ai_parser.add_argument('-M', '--model', help='Model to use', metavar='MODEL')
-    ai_parser.add_argument('-n', '--num-answers', help='Number of answers to request', type=int, default=1)
+    ai_parser.add_argument('-N', '--num-answers', help='Number of answers to request', type=int, default=1)
     ai_parser.add_argument('-m', '--max-tokens', help='Max. nr. of tokens', type=int)
     ai_parser.add_argument('-T', '--temperature', help='Temperature value', type=float)
 
@@ -148,8 +148,8 @@ def create_parser() -> argparse.ArgumentParser:
     translation_group.add_argument('-a', '--ask', nargs='+', help='Ask to translate the given text', metavar='TEXT')
     translation_group.add_argument('-c', '--create', nargs='+', help='Create a translation', metavar='TEXT')
     translation_group.add_argument('-r', '--repeat', nargs='*', help='Repeat a translation', metavar='MESSAGE')
-    translation_cmd_parser.add_argument('-S', '--source-lang', help="Source language", metavar="LANGUAGE", required=True)
-    translation_cmd_parser.add_argument('-T', '--target-lang', help="Target language", metavar="LANGUAGE", required=True)
+    translation_cmd_parser.add_argument('-l', '--source-lang', help="Source language", metavar="LANGUAGE", required=True)
+    translation_cmd_parser.add_argument('-L', '--target-lang', help="Target language", metavar="LANGUAGE", required=True)
     translation_cmd_parser.add_argument('-G', '--glossaries', nargs='+', help="List of glossary names", metavar="GLOSSARY")
     translation_cmd_parser.add_argument('-d', '--input-document', help="Document to translate", metavar="FILE")
     translation_cmd_parser.add_argument('-D', '--output-document', help="Path for the translated document", metavar="FILE")
@@ -162,12 +162,11 @@ def create_parser() -> argparse.ArgumentParser:
     glossary_group = glossary_cmd_parser.add_mutually_exclusive_group(required=True)
     glossary_group.add_argument('-c', '--create', help='Create a glossary', action='store_true')
     glossary_cmd_parser.add_argument('-n', '--name', help="Glossary name (not ID)", metavar="NAME")
-    glossary_cmd_parser.add_argument('-S', '--source-lang', help="Source language", metavar="LANGUAGE")
-    glossary_cmd_parser.add_argument('-T', '--target-lang', help="Target language", metavar="LANGUAGE")
+    glossary_cmd_parser.add_argument('-l', '--source-lang', help="Source language", metavar="LANGUAGE")
+    glossary_cmd_parser.add_argument('-L', '--target-lang', help="Target language", metavar="LANGUAGE")
     glossary_cmd_parser.add_argument('-f', '--file', help='File path of the goven glossary', metavar='GLOSSARY_FILE')
     glossary_cmd_parser.add_argument('-D', '--description', help="Glossary description", metavar="DESCRIPTION")
-    glossary_group.add_argument('-l', '--list', help='List existing glossaries', action='store_true')
-    glossary_cmd_parser.add_argument('-E', '--entries', help="Print entries when listing glossaries", action='store_true')
+    glossary_group.add_argument('-i', '--list', help='List existing glossaries', action='store_true')
 
     argcomplete.autocomplete(parser)
     return parser
@@ -221,7 +220,11 @@ def main() -> int:
     if command.func == config_cmd:
         command.func(command)
     else:
-        config = Config.from_file(args.config)
+        try:
+            config = Config.from_file(args.config)
+        except ConfigError as err:
+            print(f"{err}")
+            return 1
         create_directories(config)
         command.func(command, config)
 

tests/test_glossary.py

@@ -26,7 +26,7 @@ class TestGlossary(unittest.TestCase):
                             "  yes: sí\n"
                             "  I'm going home: me voy a casa\n")
             yaml_file_path = Path(yaml_file.name)
-
+        # create and check valid glossary
         glossary = Glossary.from_file(yaml_file_path)
         self.assertEqual(glossary.name, "Sample")
         self.assertEqual(glossary.desc, "A brief description")
@@ -55,7 +55,7 @@ class TestGlossary(unittest.TestCase):
                             "  'yes': 'sí'\n"
                             "  \"I'm going home\": 'me voy a casa'\n")
             yaml_file_path = Path(yaml_file.name)
-
+        # create and check valid glossary
         glossary = Glossary.from_file(yaml_file_path)
         self.assertEqual(glossary.name, "Sample")
         self.assertEqual(glossary.desc, "A brief description")
@@ -77,75 +77,68 @@ class TestGlossary(unittest.TestCase):
                             target_lang="fr",
                             entries={"yes": "oui"})
 
-        with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as tmp_file:
+        with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as tmp_file:
             file_path = Path(tmp_file.name)
             glossary.to_file(file_path)
-
-        with open(file_path, 'r') as file:
-            content = file.read()
-
-        self.assertIn("Name: Test", content)
-        self.assertIn("Description: Test description", content)
-        self.assertIn("ID: '666'", content)
-        self.assertIn("SourceLang: en", content)
-        self.assertIn("TargetLang: fr", content)
-        self.assertIn("Entries", content)
-        # 'yes' is a YAML keyword and therefore quoted
-        self.assertIn("'yes': oui", content)
-        file_path.unlink()  # Remove the temporary file
+            # read and check valid YAML
+            with open(file_path, 'r') as file:
+                content = file.read()
+            self.assertIn("Name: Test", content)
+            self.assertIn("Description: Test description", content)
+            self.assertIn("ID: '666'", content)
+            self.assertIn("SourceLang: en", content)
+            self.assertIn("TargetLang: fr", content)
+            self.assertIn("Entries", content)
+            # 'yes' is a YAML keyword and therefore quoted
+            self.assertIn("'yes': oui", content)
 
     def test_write_read_glossary(self) -> None:
         # Create glossary instance
         # -> use 'yes' in order to test if the YAML quoting is correctly removed when reading the file
         glossary_write = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
 
-        with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as tmp_file:
+        with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as tmp_file:
             file_path = Path(tmp_file.name)
             glossary_write.to_file(file_path)
-
-        # create new instance from glossary file
-        glossary_read = Glossary.from_file(file_path)
-        self.assertEqual(glossary_write.name, glossary_read.name)
-        self.assertEqual(glossary_write.source_lang, glossary_read.source_lang)
-        self.assertEqual(glossary_write.target_lang, glossary_read.target_lang)
-        self.assertDictEqual(glossary_write.entries, glossary_read.entries)
-
-        file_path.unlink()  # Remove the temporary file
+            # create new instance from glossary file
+            glossary_read = Glossary.from_file(file_path)
+            self.assertEqual(glossary_write.name, glossary_read.name)
+            self.assertEqual(glossary_write.source_lang, glossary_read.source_lang)
+            self.assertEqual(glossary_write.target_lang, glossary_read.target_lang)
+            self.assertDictEqual(glossary_write.entries, glossary_read.entries)
 
     def test_import_export_csv(self) -> None:
         glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={})
 
         # First export to CSV
-        with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as csvfile:
+        with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as csvfile:
             csv_file_path = Path(csvfile.name)
-        glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
-        glossary.export_csv(glossary.entries, csv_file_path)
+            glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
+            glossary.export_csv(glossary.entries, csv_file_path)
 
-        # Now import CSV
-        glossary.import_csv(csv_file_path)
-        self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
-        csv_file_path.unlink()  # Remove the temporary file
+            # Now import CSV
+            glossary.import_csv(csv_file_path)
+            self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
 
     def test_import_export_tsv(self) -> None:
         glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={})
 
         # First export to TSV
-        with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as tsvfile:
+        with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as tsvfile:
             tsv_file_path = Path(tsvfile.name)
-        glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
-        glossary.export_tsv(glossary.entries, tsv_file_path)
+            glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
+            glossary.export_tsv(glossary.entries, tsv_file_path)
 
-        # Now import TSV
-        glossary.import_tsv(tsv_file_path)
-        self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
-        tsv_file_path.unlink()  # Remove the temporary file
+            # Now import TSV
+            glossary.import_tsv(tsv_file_path)
+            self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
 
     def test_to_file_wrong_suffix(self) -> None:
         """
         Test for exception if suffix is wrong.
         """
         glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
-        with tempfile.NamedTemporaryFile('w', delete=False, suffix='.wrong') as tmp_file:
+        with tempfile.NamedTemporaryFile('w', suffix='.wrong') as tmp_file:
             file_path = Path(tmp_file.name)
             with self.assertRaises(GlossaryError) as err:
                 glossary.to_file(file_path)
@@ -156,11 +149,13 @@ class TestGlossary(unittest.TestCase):
         Test if suffix is auto-generated if omitted.
         """
         glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
-        with tempfile.NamedTemporaryFile('w', delete=False, suffix='') as tmp_file:
+        with tempfile.NamedTemporaryFile('w', suffix='') as tmp_file:
             file_path = Path(tmp_file.name)
             glossary.to_file(file_path)
-        assert glossary.file_path is not None
-        self.assertEqual(glossary.file_path.suffix, glossary_suffix)
+            assert glossary.file_path is not None
+            self.assertEqual(glossary.file_path.suffix, glossary_suffix)
+            # remove glossary file (differs from 'tmp_file' because of the added suffix
+            glossary.file_path.unlink()
 
     def test_to_str_with_id(self) -> None:
         # Create a Glossary instance with an ID
@@ -202,3 +197,13 @@ class TestGlossary(unittest.TestCase):
         self.assertIn("- An empty test glossary", glossary_str_no_id_no_entries)
         self.assertIn("- Languages: en -> fr", glossary_str_no_id_no_entries)
         self.assertIn("- Entries: 0", glossary_str_no_id_no_entries)
+
+    def test_to_str_no_description(self) -> None:
+        # Create a Glossary instance with an ID
+        glossary_with_id = Glossary(name="TestGlossary", source_lang="en", target_lang="fr",
+                                    ID="1001", entries={"one": "un"})
+        glossary_str = glossary_with_id.to_str()
+        expected_str = """TestGlossary (ID: 1001):
+- Languages: en -> fr
+- Entries: 1"""
+        self.assertEqual(expected_str, glossary_str)

tests/test_glossary_cmd.py

@@ -1,11 +1,21 @@
 import unittest
 import argparse
 import tempfile
+import io
+from contextlib import redirect_stdout
 from chatmastermind.configuration import Config
-from chatmastermind.commands.glossary import glossary_cmd, GlossaryCmdError
+from chatmastermind.commands.glossary import (
+    Glossary,
+    GlossaryCmdError,
+    glossary_cmd,
+    get_glossary_file_path,
+    create_glossary,
+    print_glossary,
+    list_glossaries
+)
 
 
-class TestGlossaryCmd(unittest.TestCase):
+class TestGlossaryCmdNoGlossaries(unittest.TestCase):
 
     def setUp(self) -> None:
         # create DB and cache
@@ -21,17 +31,119 @@ class TestGlossaryCmd(unittest.TestCase):
         self.args = argparse.Namespace(
             create=True,
             list=False,
+            print=False,
             name='new_glossary',
             file=None,
             source_lang='en',
             target_lang='de',
+            description=False,
         )
 
-    def test_glossary_cmd_no_glossaries_err(self) -> None:
-        """
-        Test calling the glossary command without a glossaries directory.
-        """
+    def test_glossary_create_no_glossaries_err(self) -> None:
         self.config.glossaries = None
         with self.assertRaises(GlossaryCmdError) as err:
-            glossary_cmd(self.args, self.config)
+            create_glossary(self.args, self.config)
             self.assertIn(str(err.exception).lower(), "glossaries directory missing")
+
+    def test_glossary_create_no_name_err(self) -> None:
+        self.args.name = None
+        with self.assertRaises(GlossaryCmdError) as err:
+            create_glossary(self.args, self.config)
+            self.assertIn(str(err.exception).lower(), "missing glossary name")
+
+    def test_glossary_create_no_source_lang_err(self) -> None:
+        self.args.source_lang = None
+        with self.assertRaises(GlossaryCmdError) as err:
+            create_glossary(self.args, self.config)
+            self.assertIn(str(err.exception).lower(), "missing source language")
+
+    def test_glossary_create_no_target_lang_err(self) -> None:
+        self.args.target_lang = None
+        with self.assertRaises(GlossaryCmdError) as err:
+            create_glossary(self.args, self.config)
+            self.assertIn(str(err.exception).lower(), "missing target language")
+
+    def test_glossary_print_no_name_err(self) -> None:
+        self.args.name = None
+        with self.assertRaises(GlossaryCmdError) as err:
+            print_glossary(self.args, self.config)
+            self.assertIn(str(err.exception).lower(), "missing glossary name")
+
+    def test_glossary_list_no_glossaries_err(self) -> None:
+        self.config.glossaries = None
+        with self.assertRaises(GlossaryCmdError) as err:
+            list_glossaries(self.args, self.config)
+            self.assertIn(str(err.exception).lower(), "glossaries directory missing")
+
+    def test_glossary_create(self) -> None:
+        self.args.create = True
+        self.args.list = False
+        self.args.print = False
+        glossary_cmd(self.args, self.config)
+        expected_path = get_glossary_file_path(self.args.name, self.config)
+        glo = Glossary.from_file(expected_path)
+        self.assertEqual(glo.name, self.args.name)
+        expected_path.unlink()
+
+    def test_glossary_create_twice_err(self) -> None:
+        self.args.create = True
+        self.args.list = False
+        self.args.print = False
+        glossary_cmd(self.args, self.config)
+        expected_path = get_glossary_file_path(self.args.name, self.config)
+        glo = Glossary.from_file(expected_path)
+        self.assertEqual(glo.name, self.args.name)
+        # create glossary with the same name again
+        with self.assertRaises(GlossaryCmdError) as err:
+            create_glossary(self.args, self.config)
+            self.assertIn(str(err.exception).lower(), "already exists")
+        expected_path.unlink()
+
+
+class TestGlossaryCmdWithGlossaries(unittest.TestCase):
+
+    def setUp(self) -> None:
+        # create DB and cache
+        self.db_dir = tempfile.TemporaryDirectory()
+        self.cache_dir = tempfile.TemporaryDirectory()
+        self.glossaries_dir = tempfile.TemporaryDirectory()
+        # create configuration
+        self.config = Config()
+        self.config.cache = self.cache_dir.name
+        self.config.db = self.db_dir.name
+        self.config.glossaries = self.glossaries_dir.name
+        # create a mock argparse.Namespace
+        self.args = argparse.Namespace(
+            create=True,
+            list=False,
+            print=False,
+            name='Glossary1',
+            file=None,
+            source_lang='en',
+            target_lang='de',
+            description=False,
+        )
+        # create Glossary1
+        glossary_cmd(self.args, self.config)
+        self.Glossary1_path = get_glossary_file_path('Glossary1', self.config)
+        # create Glossary2
+        self.args.name = 'Glossary2'
+        glossary_cmd(self.args, self.config)
+        self.Glossary2_path = get_glossary_file_path('Glossary2', self.config)
+
+    def test_glossaries_exist(self) -> None:
+        """
+        Test if the default glossaries created in setUp exist.
+        """
+        glo = Glossary.from_file(self.Glossary1_path)
+        self.assertEqual(glo.name, 'Glossary1')
+        glo = Glossary.from_file(self.Glossary2_path)
+        self.assertEqual(glo.name, 'Glossary2')
+
+    def test_glossaries_list(self) -> None:
+        self.args.create = False
+        self.args.list = True
+        with redirect_stdout(io.StringIO()) as list_output:
+            glossary_cmd(self.args, self.config)
+            self.assertIn('Glossary1', list_output.getvalue())
+            self.assertIn('Glossary2', list_output.getvalue())