From b57d78a8751f0de0c5858f475f4bed858dc2ac21 Mon Sep 17 00:00:00 2001 From: juk0de Date: Fri, 4 Aug 2023 18:15:04 +0200 Subject: [PATCH 1/4] added option '-w' to print chat history with tags --- README.md | 5 +++-- chatmastermind/main.py | 3 ++- chatmastermind/storage.py | 5 +++-- chatmastermind/utils.py | 6 +++++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 57067d2..e42d1df 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ pip install . ## Usage ```bash -cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] +cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-w] ``` ### Arguments @@ -37,6 +37,7 @@ cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMP - `-q`, `--question`: Question to ask. - `-D`, `--chat-dump`: Print chat as a Python structure. - `-d`, `--chat`: Print chat as readable text. +- `-w`, `--with-tags`: Print chat with tags. - `-c`, `--config`: Config file name (defaults to `.config.yaml`). - `-m`, `--max-tokens`: Max tokens to use. - `-T`, `--temperature`: Temperature to use. @@ -111,4 +112,4 @@ After adding this line, restart your shell or run `source argparse.ArgumentParser: parser.add_argument('-n', '--number', help='Number of answers to produce', type=int, default=1) parser.add_argument('-s', '--source', nargs='*', help='Source add content of a file to the query') parser.add_argument('-S', '--only-source-code', help='Print only source code', action='store_true') + parser.add_argument('-w', '--with-tags', help="Print chat with tags.", action='store_true') tags_arg = parser.add_argument('-t', '--tags', nargs='*', help='List of tag names', metavar='TAGS') tags_arg.completer = tags_completer # type: ignore extags_arg = parser.add_argument('-e', '--extags', nargs='*', help='List of tag names to exclude', metavar='EXTAGS') diff --git a/chatmastermind/storage.py b/chatmastermind/storage.py index 4215e5a..9fa3a84 100644 --- a/chatmastermind/storage.py +++ b/chatmastermind/storage.py @@ -62,7 +62,8 @@ def save_answers(question: str, def create_chat(question: Optional[str], tags: Optional[List[str]], extags: Optional[List[str]], - config: Dict[str, Any] + config: Dict[str, Any], + with_tags: bool = False ) -> List[Dict[str, str]]: chat: List[Dict[str, str]] = [] append_message(chat, 'system', config['system'].strip()) @@ -80,7 +81,7 @@ def create_chat(question: Optional[str], extags_do_not_match = \ not extags or not data_tags.intersection(extags) if tags_match and extags_do_not_match: - message_to_chat(data, chat) + message_to_chat(data, chat, with_tags) if question: append_message(chat, 'user', question) return chat diff --git a/chatmastermind/utils.py b/chatmastermind/utils.py index 4db8f0c..c4270c7 100644 --- a/chatmastermind/utils.py +++ b/chatmastermind/utils.py @@ -34,10 +34,14 @@ def append_message(chat: List[Dict[str, str]], def message_to_chat(message: Dict[str, str], - chat: List[Dict[str, str]] + chat: List[Dict[str, str]], + with_tags: bool = False ) -> None: append_message(chat, 'user', message['question']) append_message(chat, 'assistant', message['answer']) + if with_tags: + tags = ", ".join(message['tags']) + append_message(chat, 'tags', tags) def display_source_code(content: str) -> None: From 32bd17594b4bd9587537b225d957a1ec0859756f Mon Sep 17 00:00:00 2001 From: juk0de Date: Sat, 5 Aug 2023 08:09:35 +0200 Subject: [PATCH 2/4] Added option '-W' to print chat history with filenames --- README.md | 9 +++++---- chatmastermind/main.py | 10 ++++++---- chatmastermind/storage.py | 8 +++++--- chatmastermind/utils.py | 5 ++++- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e42d1df..74aeb9b 100644 --- a/README.md +++ b/README.md @@ -28,16 +28,17 @@ pip install . ## Usage ```bash -cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-w] +cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-w] [-W] ``` ### Arguments - `-p`, `--print`: YAML file to print. - `-q`, `--question`: Question to ask. -- `-D`, `--chat-dump`: Print chat as a Python structure. -- `-d`, `--chat`: Print chat as readable text. -- `-w`, `--with-tags`: Print chat with tags. +- `-D`, `--chat-dump`: Print chat history as a Python structure. +- `-d`, `--chat`: Print chat history as readable text. +- `-w`, `--with-tags`: Print chat history with tags. +- `-W`, `--with-tags`: Print chat history with filenames. - `-c`, `--config`: Config file name (defaults to `.config.yaml`). - `-m`, `--max-tokens`: Max tokens to use. - `-T`, `--temperature`: Temperature to use. diff --git a/chatmastermind/main.py b/chatmastermind/main.py index 011577d..9828399 100755 --- a/chatmastermind/main.py +++ b/chatmastermind/main.py @@ -55,7 +55,8 @@ def process_and_display_chat(args: argparse.Namespace, question_parts.append(f"```\n{r.read().strip()}\n```") full_question = '\n\n'.join(question_parts) - chat = create_chat(full_question, tags, extags, config, args.with_tags) + chat = create_chat(full_question, tags, extags, config, + args.with_tags, args.with_file) display_chat(chat, dump, args.only_source_code) return chat, full_question, tags @@ -85,8 +86,8 @@ def create_parser() -> argparse.ArgumentParser: group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-p', '--print', help='File to print') group.add_argument('-q', '--question', nargs='*', help='Question to ask') - group.add_argument('-D', '--chat-dump', help="Print chat as Python structure", action='store_true') - group.add_argument('-d', '--chat', help="Print chat as readable text", action='store_true') + group.add_argument('-D', '--chat-dump', help="Print chat history as Python structure", action='store_true') + group.add_argument('-d', '--chat', help="Print chat history as readable text", action='store_true') parser.add_argument('-c', '--config', help='Config file name.', default=default_config) parser.add_argument('-m', '--max-tokens', help='Max tokens to use', type=int) parser.add_argument('-T', '--temperature', help='Temperature to use', type=float) @@ -94,7 +95,8 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument('-n', '--number', help='Number of answers to produce', type=int, default=1) parser.add_argument('-s', '--source', nargs='*', help='Source add content of a file to the query') parser.add_argument('-S', '--only-source-code', help='Print only source code', action='store_true') - parser.add_argument('-w', '--with-tags', help="Print chat with tags.", action='store_true') + parser.add_argument('-w', '--with-tags', help="Print chat history with tags.", action='store_true') + parser.add_argument('-W', '--with-file', help="Print chat history with filename.", action='store_true') tags_arg = parser.add_argument('-t', '--tags', nargs='*', help='List of tag names', metavar='TAGS') tags_arg.completer = tags_completer # type: ignore extags_arg = parser.add_argument('-e', '--extags', nargs='*', help='List of tag names to exclude', metavar='EXTAGS') diff --git a/chatmastermind/storage.py b/chatmastermind/storage.py index 9fa3a84..7418edf 100644 --- a/chatmastermind/storage.py +++ b/chatmastermind/storage.py @@ -15,7 +15,8 @@ def read_file(fname: str, tags_only: bool = False) -> Dict[str, Any]: answer_idx = text.index("==== ANSWER ====") question = "\n".join(text[question_idx:answer_idx]).strip() answer = "\n".join(text[answer_idx + 1:]).strip() - return {"question": question, "answer": answer, "tags": tags} + return {"question": question, "answer": answer, "tags": tags, + "file": pathlib.Path(fname).name} def dump_data(data: Dict[str, Any]) -> str: @@ -63,7 +64,8 @@ def create_chat(question: Optional[str], tags: Optional[List[str]], extags: Optional[List[str]], config: Dict[str, Any], - with_tags: bool = False + with_tags: bool = False, + with_file: bool = False ) -> List[Dict[str, str]]: chat: List[Dict[str, str]] = [] append_message(chat, 'system', config['system'].strip()) @@ -81,7 +83,7 @@ def create_chat(question: Optional[str], extags_do_not_match = \ not extags or not data_tags.intersection(extags) if tags_match and extags_do_not_match: - message_to_chat(data, chat, with_tags) + message_to_chat(data, chat, with_tags, with_file) if question: append_message(chat, 'user', question) return chat diff --git a/chatmastermind/utils.py b/chatmastermind/utils.py index c4270c7..ef876e8 100644 --- a/chatmastermind/utils.py +++ b/chatmastermind/utils.py @@ -35,13 +35,16 @@ def append_message(chat: List[Dict[str, str]], def message_to_chat(message: Dict[str, str], chat: List[Dict[str, str]], - with_tags: bool = False + with_tags: bool = False, + with_file: bool = False ) -> None: append_message(chat, 'user', message['question']) append_message(chat, 'assistant', message['answer']) if with_tags: tags = ", ".join(message['tags']) append_message(chat, 'tags', tags) + if with_file: + append_message(chat, 'file', message['file']) def display_source_code(content: str) -> None: From 5d55bed0ec7ba33ab175b92da113705a49ae69c3 Mon Sep 17 00:00:00 2001 From: juk0de Date: Sat, 5 Aug 2023 11:16:06 +0200 Subject: [PATCH 3/4] added option '-l' to list all tags and their frequency --- README.md | 3 ++- chatmastermind/main.py | 15 ++++++++++++--- chatmastermind/storage.py | 5 ++++- chatmastermind/utils.py | 7 +++++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 74aeb9b..56e4525 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ pip install . ## Usage ```bash -cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-w] [-W] +cmm [-h] [-p PRINT | -q QUESTION | -D | -d | -l] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-w] [-W] ``` ### Arguments @@ -39,6 +39,7 @@ cmm [-h] [-p PRINT | -q QUESTION | -D | -d] [-c CONFIG] [-m MAX_TOKENS] [-T TEMP - `-d`, `--chat`: Print chat history as readable text. - `-w`, `--with-tags`: Print chat history with tags. - `-W`, `--with-tags`: Print chat history with filenames. +- `-l`, `--list-tags`: List all tags and their frequency. - `-c`, `--config`: Config file name (defaults to `.config.yaml`). - `-m`, `--max-tokens`: Max tokens to use. - `-T`, `--temperature`: Temperature to use. diff --git a/chatmastermind/main.py b/chatmastermind/main.py index 9828399..5dad76a 100755 --- a/chatmastermind/main.py +++ b/chatmastermind/main.py @@ -7,8 +7,8 @@ import sys import argcomplete import argparse import pathlib -from .utils import terminal_width, process_tags, display_chat, display_source_code -from .storage import save_answers, create_chat, get_tags, read_file, dump_data +from .utils import terminal_width, process_tags, display_chat, display_source_code, display_tags_frequency +from .storage import save_answers, create_chat, get_tags, get_tags_unique, read_file, dump_data from .api_client import ai, openai_api_key from itertools import zip_longest @@ -60,6 +60,12 @@ def process_and_display_chat(args: argparse.Namespace, display_chat(chat, dump, args.only_source_code) return chat, full_question, tags +def process_and_display_tags(args: argparse.Namespace, + config: dict, + dump: bool=False + ) -> None: + display_tags_frequency(get_tags(config, None), dump) + def handle_question(args: argparse.Namespace, config: dict, @@ -76,7 +82,7 @@ def handle_question(args: argparse.Namespace, def tags_completer(prefix, parsed_args, **kwargs): with open(parsed_args.config, 'r') as f: config = yaml.load(f, Loader=yaml.FullLoader) - return get_tags(config, prefix) + return get_tags_unique(config, prefix) def create_parser() -> argparse.ArgumentParser: @@ -88,6 +94,7 @@ def create_parser() -> argparse.ArgumentParser: group.add_argument('-q', '--question', nargs='*', help='Question to ask') group.add_argument('-D', '--chat-dump', help="Print chat history as Python structure", action='store_true') group.add_argument('-d', '--chat', help="Print chat history as readable text", action='store_true') + group.add_argument('-l', '--list-tags', help="List all tags and their frequency", action='store_true') parser.add_argument('-c', '--config', help='Config file name.', default=default_config) parser.add_argument('-m', '--max-tokens', help='Max tokens to use', type=int) parser.add_argument('-T', '--temperature', help='Temperature to use', type=float) @@ -133,6 +140,8 @@ def main() -> int: process_and_display_chat(args, config, dump=True) elif args.chat: process_and_display_chat(args, config) + elif args.list_tags: + process_and_display_tags(args, config) return 0 diff --git a/chatmastermind/storage.py b/chatmastermind/storage.py index 7418edf..0839d61 100644 --- a/chatmastermind/storage.py +++ b/chatmastermind/storage.py @@ -105,4 +105,7 @@ def get_tags(config: Dict[str, Any], prefix: Optional[str]) -> List[str]: result.append(tag) else: result.append(tag) - return list(set(result)) + return result + +def get_tags_unique(config: Dict[str, Any], prefix: Optional[str]) -> List[str]: + return list(set(get_tags(config, prefix))) diff --git a/chatmastermind/utils.py b/chatmastermind/utils.py index ef876e8..2dceb69 100644 --- a/chatmastermind/utils.py +++ b/chatmastermind/utils.py @@ -73,3 +73,10 @@ def display_chat(chat, dump=False, source_code=False) -> None: print(message['content']) else: print(f"{message['role'].upper()}: {message['content']}") + +def display_tags_frequency(tags: List[str], dump=False) -> None: + if dump: + pp(tags) + return + for tag in set(tags): + print(f"-{tag} : {tags.count(tag)}") From 8e4a02b932c9fa3ae98518366a7e0d3b79da22ac Mon Sep 17 00:00:00 2001 From: juk0de Date: Sat, 5 Aug 2023 11:41:36 +0200 Subject: [PATCH 4/4] added option '-a' to match all tags when selecting chat history entries --- README.md | 3 ++- chatmastermind/main.py | 4 +++- chatmastermind/storage.py | 8 ++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 56e4525..dd38a71 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ pip install . ## Usage ```bash -cmm [-h] [-p PRINT | -q QUESTION | -D | -d | -l] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-w] [-W] +cmm [-h] [-p PRINT | -q QUESTION | -D | -d | -l] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-a] [-w] [-W] ``` ### Arguments @@ -37,6 +37,7 @@ cmm [-h] [-p PRINT | -q QUESTION | -D | -d | -l] [-c CONFIG] [-m MAX_TOKENS] [-T - `-q`, `--question`: Question to ask. - `-D`, `--chat-dump`: Print chat history as a Python structure. - `-d`, `--chat`: Print chat history as readable text. +- `-a`, `--match-all-tags`: All given tags must match when selecting chat history entries. - `-w`, `--with-tags`: Print chat history with tags. - `-W`, `--with-tags`: Print chat history with filenames. - `-l`, `--list-tags`: List all tags and their frequency. diff --git a/chatmastermind/main.py b/chatmastermind/main.py index 5dad76a..6fe4984 100755 --- a/chatmastermind/main.py +++ b/chatmastermind/main.py @@ -56,7 +56,8 @@ def process_and_display_chat(args: argparse.Namespace, full_question = '\n\n'.join(question_parts) chat = create_chat(full_question, tags, extags, config, - args.with_tags, args.with_file) + args.match_all_tags, args.with_tags, + args.with_file) display_chat(chat, dump, args.only_source_code) return chat, full_question, tags @@ -104,6 +105,7 @@ def create_parser() -> argparse.ArgumentParser: parser.add_argument('-S', '--only-source-code', help='Print only source code', action='store_true') parser.add_argument('-w', '--with-tags', help="Print chat history with tags.", action='store_true') parser.add_argument('-W', '--with-file', help="Print chat history with filename.", action='store_true') + parser.add_argument('-a', '--match-all-tags', help="All given tags must match when selecting chat history entries.", action='store_true') tags_arg = parser.add_argument('-t', '--tags', nargs='*', help='List of tag names', metavar='TAGS') tags_arg.completer = tags_completer # type: ignore extags_arg = parser.add_argument('-e', '--extags', nargs='*', help='List of tag names to exclude', metavar='EXTAGS') diff --git a/chatmastermind/storage.py b/chatmastermind/storage.py index 0839d61..ee1443f 100644 --- a/chatmastermind/storage.py +++ b/chatmastermind/storage.py @@ -64,6 +64,7 @@ def create_chat(question: Optional[str], tags: Optional[List[str]], extags: Optional[List[str]], config: Dict[str, Any], + match_all_tags: bool = False, with_tags: bool = False, with_file: bool = False ) -> List[Dict[str, str]]: @@ -78,8 +79,11 @@ def create_chat(question: Optional[str], else: continue data_tags = set(data.get('tags', [])) - tags_match = \ - not tags or data_tags.intersection(tags) + tags_match: bool + if match_all_tags: + tags_match = not tags or set(tags).issubset(data_tags) + else: + tags_match = not tags or bool(data_tags.intersection(tags)) extags_do_not_match = \ not extags or not data_tags.intersection(extags) if tags_match and extags_do_not_match: