From fa19c232c4c727155a45f8482206aa33d7b243ac Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 02:36:06 +0300 Subject: [PATCH 01/11] Remove toolforge_page module and related tests Deleted the toolforge_page implementation, its import wrapper, and associated test file. This change removes unused or deprecated functionality related to toolforge page handling from the codebase. --- newapi/pages_bots/toolforge_page.py | 77 ----------------------------- newapi/toolforge_page.py | 20 -------- z_te_sts/test_toolforge_page.py | 55 --------------------- 3 files changed, 152 deletions(-) delete mode 100644 newapi/pages_bots/toolforge_page.py delete mode 100644 newapi/toolforge_page.py delete mode 100644 z_te_sts/test_toolforge_page.py diff --git a/newapi/pages_bots/toolforge_page.py b/newapi/pages_bots/toolforge_page.py deleted file mode 100644 index 488dff7..0000000 --- a/newapi/pages_bots/toolforge_page.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -from newapi import toolforge_page -# --- -User_tables_md = { - "username": medwiki_account.username, - "password": medwiki_account.password, -} -# --- -toolforge_page.add_User_table(User_tables_md, "toolforge", "medwiki") -# --- -CatDepth = toolforge_page.subcatquery -MainPage = toolforge_page.MainPage(title, lang, family="toolforge") -# --- -""" -# --- -from ..super.S_Category import catdepth_new - -from ..super.S_API import bot_api -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap - -User_tables = {} -logins_cache = {} - - -def add_User_table(table, family, lang): - # --- - User_tables[(lang, family)] = table - User_tables[("*", family)] = table - - -def log_it(lang, family): - # --- - table = User_tables.get((lang, family)) or User_tables.get(("*", family)) - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, table) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - - -def MainPage(title, lang, family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode="", family="wikipedia", **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result - - -__all__ = [ - # "bot_api", - # "super_page", - "catdepth_new", - "MainPage", - "add_User_table", - "CatDepth", -] diff --git a/newapi/toolforge_page.py b/newapi/toolforge_page.py deleted file mode 100644 index 61cba10..0000000 --- a/newapi/toolforge_page.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -""" -from .pages_bots.toolforge_page import ( - # bot_api, - # super_page, - catdepth_new, - MainPage, - add_User_table, - CatDepth, -) - -__all__ = [ - # "bot_api", - # "super_page", - "catdepth_new", - "MainPage", - "add_User_table", - "CatDepth", -] diff --git a/z_te_sts/test_toolforge_page.py b/z_te_sts/test_toolforge_page.py deleted file mode 100644 index 67dad18..0000000 --- a/z_te_sts/test_toolforge_page.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.path.append("I:/mdwiki/pybot") -sys.argv.append("ask") -from newapi import toolforge_page - -from copy_to_en.bots import medwiki_account - -User_tables_md = { - "username": medwiki_account.username, - "password": medwiki_account.password, -} -# --- -User_tables_mdcx = { - "username": medwiki_account.username_cx, - "password": medwiki_account.password_cx, -} -# --- -toolforge_page.add_User_table(User_tables_md, "toolforge", "medwiki") -# --- -toolforge_page.add_User_table(User_tables_mdcx, "toolforge", "mdwikicx") -# --- -CatDepth = toolforge_page.CatDepth -MainPage = toolforge_page.MainPage - -lists = { - "page" : "Main_Page", - "cat" : "Category:Translations" -} -# --- -for site in ["medwiki", "mdwikicx"]: - # --- - print(f"___________________{site=}___________________") - # --- - print("//////// page ////////") - # --- - title = "Main_Page" - Category = "Category:Translations" - # --- - print(f"{site=}, {title=}") - # --- - page = MainPage(title, site, family="toolforge") - # --- - text = page.get_text() - print(f"{len(text)=}") - # --- - print("//////// Category ////////") - # --- - members = CatDepth(Category, sitecode=site, family="toolforge", ns=14) - # --- - print(members.keys()) - print(f"{site=} {Category=}, members: {len(members)}") From d471861faaaaf29fe446f5fc1200bc119b513014 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:02:25 +0300 Subject: [PATCH 02/11] Remove NCC and MDWiki page modules and related files Deleted modules and files related to NCC and MDWiki page handling, including user account configuration, page bots, and test scripts. Updated Diagram.md to remove reference to ncc_page.py and cleaned up catdepth_new.py by removing an unused import. This streamlines the codebase by eliminating deprecated or redundant components. --- Diagram.md | 1 - README1.md | 113 ------------------------ newapi/accounts/user_account_ncc.py | 30 ------- newapi/mdwiki_page.py | 21 ----- newapi/ncc_page.py | 21 ----- newapi/pages_bots/mdwiki_page.py | 49 ---------- newapi/pages_bots/ncc_page.py | 96 -------------------- newapi/super/S_Category/catdepth_new.py | 1 - z_te_sts/test_ncc_page.py | 23 ----- 9 files changed, 355 deletions(-) delete mode 100644 README1.md delete mode 100644 newapi/accounts/user_account_ncc.py delete mode 100644 newapi/mdwiki_page.py delete mode 100644 newapi/ncc_page.py delete mode 100644 newapi/pages_bots/mdwiki_page.py delete mode 100644 newapi/pages_bots/ncc_page.py delete mode 100644 z_te_sts/test_ncc_page.py diff --git a/Diagram.md b/Diagram.md index 200dfae..d25345b 100644 --- a/Diagram.md +++ b/Diagram.md @@ -34,7 +34,6 @@ flowchart TD click PageOps "https://github.com/wikiar/newapi/blob/main/page.py" click PageOps "https://github.com/wikiar/newapi/blob/main/wiki_page.py" click PageOps "https://github.com/wikiar/newapi/blob/main/mdwiki_page.py" - click PageOps "https://github.com/wikiar/newapi/blob/main/ncc_page.py" click PageOps "https://github.com/wikiar/newapi/tree/main/super/page_bots/" %% Click Events for Bot Functionality Modules diff --git a/README1.md b/README1.md deleted file mode 100644 index 1ac0e14..0000000 --- a/README1.md +++ /dev/null @@ -1,113 +0,0 @@ -# Wikimedia API Python Module - -## Overview -This project is a Python module designed to interact with the Wikimedia API. It provides a unified, programmable interface for common Wikimedia actions such as: -- Logging in -- Retrieving and editing pages -- Working with categories and templates -- Performing various Wikimedia API actions - -The module abstracts the Wikimedia API into user-friendly functions and classes, reducing the need for direct HTTP request handling and token management. - -## Features -- **API Interface Layer**: Encapsulates core Wikimedia interactions, handling requests and responses. -- **Login Module**: Simplifies authentication with multiple bot-based login implementations. -- **Page Management Module**: Offers functionality to check page existence, edit pages, and manage page elements. -- **Category and Template Handling**: Supports category depth processing and template handling. -- **Bot Modules**: Automates API interactions and page operations. -- **MWClient Wrappers**: Provides lower-level API communication support. -- **Database & Utility Bots**: Manages database interactions and caching. -- **Testing & CI/CD Support**: Includes a test suite and GitHub workflows for automated validation. - -## System Architecture -The project follows a modular and layered architecture: -1. **Client/Consumer Applications** interact with the module. -2. **API Interface Layer** (e.g., `NEW_API` class) abstracts direct Wikimedia API calls. -3. **Login Module** handles session authentication. -4. **Page Management Module** enables page retrieval, editing, and processing. -5. **Category and Template Handling** supports Wikimedia-specific structures. -6. **Bot Modules** automate repetitive Wikimedia tasks. -7. **MWClient Wrappers** facilitate low-level API interactions. -8. **Database & Utility Bots** provide storage and session support. -9. **Testing & CI/CD** ensures reliability through automated testing and configuration management. - -## File Mapping -### API Interface Layer -- `wiki_page.py`, `page.py`, `mdwiki_page.py`, `ncc_page.py` -- `super/super_page.py` - -### Login Module -- `super/super_login.py` -- `super/S_Login/` (including `bot.py`, `bot_new.py`, `cookies_bot.py`) - -### Page Management Module -- `wiki_page.py`, `page.py`, `mdwiki_page.py`, `ncc_page.py` -- `super/super_page.py` - -### Category and Template Handling -- `super/catdepth_new.py` - -### Bot Modules -- API bots: `super/botapi_bots/` -- Page bots: `super/page_bots/` -- Additional bot utilities: `botEdit.py`, `super/bots/` - -### MWClient Wrappers -- `super/S_Login/mwclient/` (including `client.py`, `page.py`, `errors.py`, `image.py`, `listing.py`, `sleep.py`, `util.py`) - -### Database & Utility Bots -- `db_bot.py`, `pymysql_bot.py` - -### Testing Suite -- `tests/` (e.g., `tests/test_bot_api.py`, `tests/test_db_bot.py`) - -### CI/CD & Configuration -- `.github/` (GitHub workflows and issue templates) -- YAML configuration: `sweep.yaml`, `.coderabbit.yaml` -- Dependencies: `requirements.in` - -## System Design Diagram Guidelines -To accurately visualize the architecture: -1. **Identify Main Components**: Include API layers, login, page management, bots, and utility modules. -2. **Map Relationships**: - - Client → Login Module → MWClient Wrapper → API Interface → Page Management - - Bots interact with API Interface and Page Management -3. **Use Layered Representation**: - - High-level API components (NEW_API, MainPage) - - Mid-level modules (bot handlers, database utilities) - - Low-level API wrappers (MWClient interactions) -4. **Diagram Elements**: - - Labeled boxes for components - - Directional arrows to indicate data flow - - Different colors for API layers, bots, and utility modules - - Sub-diagrams for clusters like the `super` directory - -## Installation & Usage -### Installation -```sh -pip install -r requirements.in -``` - -### Usage Example -```python -from newapi.page import MainPage - -# Initialize API -page = MainPage("Example_Page") - -# Check if the page exists -if page.exists(): - print("Page found!") - -# Edit a page -page.edit("Updated content") -``` - -## Contribution -1. Fork the repository -2. Create a new branch -3. Make changes and commit -4. Submit a pull request - -## License -This project is licensed under the MIT License. diff --git a/newapi/accounts/user_account_ncc.py b/newapi/accounts/user_account_ncc.py deleted file mode 100644 index 7a63f4e..0000000 --- a/newapi/accounts/user_account_ncc.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -# --- -from .accounts import user_account_ncc -# --- -User_tables = { - "username": user_account_ncc.username, - "password": user_account_ncc.password, -} -# --- -""" - -import os -import configparser - -home_dir = os.getenv("HOME") -project = home_dir if home_dir else "I:/ncc" -# --- -config = configparser.ConfigParser() -config.read(f"{project}/confs/nccommons_user.ini") -# --- -username = config["DEFAULT"].get("username", "").strip() -password = config["DEFAULT"].get("password", "").strip() - -User_tables = { - "username": username, - "password": password, -} - -SITECODE = "www" -FAMILY = "nccommons" diff --git a/newapi/mdwiki_page.py b/newapi/mdwiki_page.py deleted file mode 100644 index e79eb83..0000000 --- a/newapi/mdwiki_page.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -""" -from .pages_bots.mdwiki_page import ( - home_dir, - MainPage, - md_MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes -) - -__all__ = [ - "home_dir", - "user_agent", - "MainPage", - "md_MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/newapi/ncc_page.py b/newapi/ncc_page.py deleted file mode 100644 index 0677482..0000000 --- a/newapi/ncc_page.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -""" -from .pages_bots.ncc_page import ( - home_dir, - ncc_MainPage, - MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes -) - -__all__ = [ - "home_dir", - "user_agent", - "ncc_MainPage", - "MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/newapi/pages_bots/mdwiki_page.py b/newapi/pages_bots/mdwiki_page.py deleted file mode 100644 index cd56b09..0000000 --- a/newapi/pages_bots/mdwiki_page.py +++ /dev/null @@ -1,49 +0,0 @@ -# --- -""" -""" -# --- -import os -import functools -import sys - -if "mwclient" not in sys.argv: - sys.argv.append("nomwclient") - -from .all_apis import ALL_APIS -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes -from ..accounts.user_account_new import User_tables - -user_agent = default_user_agent() - -change_codes = lang_codes.change_codes - -home_dir = os.getenv("HOME") - - -@functools.lru_cache(maxsize=1) -def load_main_api() -> ALL_APIS: - return ALL_APIS( - lang="www", - family="mdwiki", - username=User_tables["username"], - password=User_tables["password"], - ) - - -main_api = load_main_api() - -NEW_API = main_api.NEW_API -MainPage = main_api.MainPage -CatDepth = main_api.CatDepth -md_MainPage = MainPage - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'md_MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots/ncc_page.py b/newapi/pages_bots/ncc_page.py deleted file mode 100644 index 9c92285..0000000 --- a/newapi/pages_bots/ncc_page.py +++ /dev/null @@ -1,96 +0,0 @@ -# --- -""" - -from newapi.ncc_page import CatDepth - -# cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, ns=10, nslist=[], onlyns=False, tempyes=[]) - -from newapi.ncc_page import MainPage as ncc_MainPage -''' -page = ncc_MainPage(title, 'www', family='nccommons') -exists = page.exists() -if not exists: return -# --- -''' - - -from newapi.ncc_page import NEW_API -# api_new = NEW_API('www', family='nccommons') -# login = api_new.Login_to_wiki() -# json1 = api_new.post_params(params, addtoken=False) -# pages = api_new.Find_pages_exists_or_not(liste) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# revisions= api_new.get_revisions(title) - -""" - -import os -# import sys -# --- -home_dir = os.getenv("HOME") -# --- -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes - -from ..accounts.user_account_ncc import User_tables, SITECODE, FAMILY -# --- -user_agent = default_user_agent() -# --- -change_codes = lang_codes.change_codes - -logins_cache = {} - -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - -def MainPage(title, lang, family=FAMILY): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - -def CatDepth(title, sitecode=SITECODE, family=FAMILY, **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result - - -ncc_MainPage = MainPage - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'ncc_MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/super/S_Category/catdepth_new.py b/newapi/super/S_Category/catdepth_new.py index fae05e4..da083a3 100644 --- a/newapi/super/S_Category/catdepth_new.py +++ b/newapi/super/S_Category/catdepth_new.py @@ -1,6 +1,5 @@ """ -from newapi.ncc_page import CatDepth # cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, ns=10, nslist=[], onlyns=False, tempyes=[]) """ diff --git a/z_te_sts/test_ncc_page.py b/z_te_sts/test_ncc_page.py deleted file mode 100644 index 62cf840..0000000 --- a/z_te_sts/test_ncc_page.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.ncc_page import MainPage, CatDepth - -title = "Category:Pages_with_script_errors" - -# cat_members = CatDepth(title, depth=0, ns="10", nslist=[], tempyes=[]) - -cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, onlyns=10) - -# print(cat_members) -print(f"{len(cat_members)=}") - -page = MainPage("Bilateral mesial temporal polymicrogyria (Radiopaedia 76456-88181 Axial SWI)", "www", family="nccommons") -# --- -text = page.get_text() -print(f"{len(text)=}") From 2807c3300cf10a3ae4862c5d9fabf9b3881bd659 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:03:24 +0300 Subject: [PATCH 03/11] Update catdepth_new.py --- newapi/super/S_Category/catdepth_new.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/newapi/super/S_Category/catdepth_new.py b/newapi/super/S_Category/catdepth_new.py index da083a3..2679d07 100644 --- a/newapi/super/S_Category/catdepth_new.py +++ b/newapi/super/S_Category/catdepth_new.py @@ -1,7 +1,5 @@ """ -# cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, ns=10, nslist=[], onlyns=False, tempyes=[]) - """ import time import sys From 510d2692441d87f50de850e7066238e008d7a629 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:07:17 +0300 Subject: [PATCH 04/11] 1 --- z_te_sts/__init__.py | 1 - z_te_sts/nc_files.db | Bin 8192 -> 0 bytes z_te_sts/test_bot_api.py | 254 ------------------------------------- z_te_sts/test_bot_edit.py | 22 ---- z_te_sts/test_catdepth.py | 22 ---- z_te_sts/test_db_bot.py | 83 ------------ z_te_sts/test_langs.py | 27 ---- z_te_sts/test_login.py | 26 ---- z_te_sts/test_login1.py | 30 ----- z_te_sts/test_page.py | 76 ----------- z_te_sts/test_runner.py | 101 --------------- z_te_sts/test_wiki_page.py | 63 --------- 12 files changed, 705 deletions(-) delete mode 100644 z_te_sts/__init__.py delete mode 100644 z_te_sts/nc_files.db delete mode 100644 z_te_sts/test_bot_api.py delete mode 100644 z_te_sts/test_bot_edit.py delete mode 100644 z_te_sts/test_catdepth.py delete mode 100644 z_te_sts/test_db_bot.py delete mode 100644 z_te_sts/test_langs.py delete mode 100644 z_te_sts/test_login.py delete mode 100644 z_te_sts/test_login1.py delete mode 100644 z_te_sts/test_page.py delete mode 100644 z_te_sts/test_runner.py delete mode 100644 z_te_sts/test_wiki_page.py diff --git a/z_te_sts/__init__.py b/z_te_sts/__init__.py deleted file mode 100644 index 40a96af..0000000 --- a/z_te_sts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/z_te_sts/nc_files.db b/z_te_sts/nc_files.db deleted file mode 100644 index 987fe973a0c2e1e2a3405db3606ad26367503aee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeI$K}*9h7zW^^lPLqKw;?k46a@!PYGt0Sn#r(=b`czF&*y$e6d6b&Dy<2a*Tw!}DR`$UYfikX&~WpVQ|S2kwN``@a< zw(s8;O^VwgGXn(y2tWV=5P$##AOHafKmY;|_-_I)Yka=mXzFj_b(eW;QNlpRS%YI|_hk2kN009U<00Izz00bZa0SG_<0ucBs z0_#?lEjJfdJ5zI~aMz2+k6Mw;Zve=39k<> start def number {n}, name:{name}:", p=True) - # --- - def_name = func.__doc__ - printe.output(f"<> test: {def_name}:", p=True) - # --- - if "tat" in sys.argv: - continue - # --- - result = func() - # --- - # printe.output( result ) - # --- - if isinstance(result, dict): - for n, (na, ta) in enumerate(result.items()): - na2 = na # f" '{na}' ".ljust(10) - # --- - if na == "claims": - for x, u in ta.items(): - ta = {x: u} - break - # --- - # ta = json.dumps(ta, indent=2, ensure_ascii=False) - # --- - printe.output(f"{n}: {na2}: {ta}") - # --- - if result == "": - raise Exception("result == ''") - # --- - if "printresult" in sys.argv: - if isinstance(result, str): - printe.output(f"result:{result}") - elif isinstance(result, list): - printe.output(result) - else: - printe.output(result) - else: - printe.output("<> add 'printresult' to sys.argv to print result") - # --- - if result: - printe.output(f"{len(result)=}", p=True) - # --- - printe.output("=====================") - printe.output(f"<> test: {def_name} end...") - printe.output("time.sleep(1)") - time.sleep(1) - - -if __name__ == "__main__": - testmybot().start() diff --git a/z_te_sts/test_bot_edit.py b/z_te_sts/test_bot_edit.py deleted file mode 100644 index aad5a38..0000000 --- a/z_te_sts/test_bot_edit.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -from newapi.page import MainPage - -# --- -titles = [ - "أضرحة عائلة ميرزا أديغوزال بك", - "موسى والراعي (قصة)", - "جبل عمر", - "ويكيبيديا:ملعب", - "القدس خلال فترة الهيكل الثاني", - "أضرحة عائلة ميرزا أديغوزال بك", -] -# --- -for x in titles: - page = MainPage(x, "ar") - - canedit = page.can_edit(delay=30) - - # print(f"Page: {x}, \t:{canedit=}") - # break diff --git a/z_te_sts/test_catdepth.py b/z_te_sts/test_catdepth.py deleted file mode 100644 index a2dc9db..0000000 --- a/z_te_sts/test_catdepth.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys -# sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.page import CatDepth - -cat_members = CatDepth("اليمن", sitecode='ar', family="wikipedia", depth=0, ns="all", nslist=[], tempyes=[]) - -# print(cat_members.keys()) -print(f"xxxxxxxxxxxxxx: {len(cat_members)=}") - -cat_members2 = CatDepth("Yemen", sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], tempyes=[]) - -# print(cat_members2.keys()) -print(f"xxxxxxxxxxxxxx: {len(cat_members2)=}") - -cat_members3 = CatDepth("صنعاء", sitecode='ar', family="wikipedia", depth=0, ns="all", nslist=[], tempyes=[]) - -# print(cat_members3.keys()) -print(f"xxxxxxxxxxxxxx: {len(cat_members3)=}") diff --git a/z_te_sts/test_db_bot.py b/z_te_sts/test_db_bot.py deleted file mode 100644 index 0c79800..0000000 --- a/z_te_sts/test_db_bot.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import os -import sys -from datetime import datetime -from pathlib import Path -Dir = Path(__file__).resolve().parent - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi import db_bot - -LiteDB = db_bot.LiteDB - -def test(): - db_path = Dir / "nc_files.db" - if not os.path.exists(db_path): - print(f"Creating {db_path}") - db_path.touch() - - nc_files_db = LiteDB(db_path) - - nc_files_db.create_table( - "nc_files", - {"id": int, "lang": str, "title": str, "views": int, "dat": str}, - pk="id", - defaults={"views": 0}, - ) - - nc_files_db.show_tables() - - # Insert sample data - nc_files_db.insert( - "nc_files", - { - "lang": "English", - "title": "Sample Title 1", - # "views": 100, - "dat": datetime.now().strftime("%Y-%m-%d"), - }, - ) - - nc_files_db.insert( - "nc_files", - { - "lang": "French", - "title": "Sample Title 2", - # "views": 200, - "dat": datetime.now().strftime("%Y-%m-%d"), - }, - ) - - # Retrieve data - data = nc_files_db.get_data("nc_files") - for row in data: - print(row) - - -def test2(): - db_path = "/data/mdwiki/public_html/ncc/Tables/nc_files.db" - if not os.path.exists(db_path): - db_path = "I:/mdwiki/pybot/ncc_core/nc_import/bots/nc_files.db" - - nc_files_db = LiteDB(db_path) - - print("________") - # Select data - # data = nc_files_db.select("nc_files", {"lang": "English"}) - # print(data) - # for row in data: - # print(row) - - # Retrieve data - data = nc_files_db.get_data("nc_files") - for row in data: - print(row) - - -if __name__ == "__main__": - test() - test2() diff --git a/z_te_sts/test_langs.py b/z_te_sts/test_langs.py deleted file mode 100644 index edaf4da..0000000 --- a/z_te_sts/test_langs.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.page import MainPage - -page = MainPage('وب:ملعب', 'ar') -exists = page.exists() - -# --- -print('--------------') -print('simple:') -from newapi.page import NEW_API - -# --- -en_api_new = NEW_API('simple', family='wikipedia') -# --- -# en_api_new.Login_to_wiki() -# --- -pages = en_api_new.Find_pages_exists_or_not(['yemen']) -# --- -print('--------------') -save_page = page.save(newtext='test!', summary='', nocreate=1, minor='') diff --git a/z_te_sts/test_login.py b/z_te_sts/test_login.py deleted file mode 100644 index 7fd9114..0000000 --- a/z_te_sts/test_login.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.accounts.useraccount import User_tables_bot -from newapi.super import super_login - -User_tables = User_tables_bot -# --- - -Login = super_login.Login -# --- -bot = Login("ar", family="wikipedia") -# --- -bot.add_User_tables("wikipedia", User_tables) -# --- -params = {"action": "query", "titles": f"User:{User_tables['username']}", "prop": "revisions", "rvprop": "content", "rvslots": "*", "format": "json"} -# --- -json1 = bot.post(params, Type="post", addtoken=False) -# --- -# print(json1) - -print(f"{len(json1)=}") diff --git a/z_te_sts/test_login1.py b/z_te_sts/test_login1.py deleted file mode 100644 index 3f92dea..0000000 --- a/z_te_sts/test_login1.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.super import super_login - -from newapi.accounts.useraccount import User_tables_bot - -User_tables = User_tables_bot -User_tables["password"] += "123" -# --- - -# super_login.User_tables["wikipedia"] = User_tables - -Login = super_login.Login -# --- -bot = Login("simple", family="wikipedia") -login = bot.Log_to_wiki() -# --- -params = {"action": "query", "titles": "User:Mr. Ibrahem", "prop": "revisions", "rvprop": "content", "rvslots": "*", "format": "json"} -# --- -json1 = bot.post(params, Type="post", addtoken=False) -# --- -print(json1) - -print(f"{len(json1)=}") -# --- diff --git a/z_te_sts/test_page.py b/z_te_sts/test_page.py deleted file mode 100644 index ec151dc..0000000 --- a/z_te_sts/test_page.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys -# sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.page import MainPage - -page = MainPage("وبxx:ملعب", "ar") -print(page.exists()) - -pageen = MainPage("الصفحة الرxئيسة", 'ar') -print(pageen.exists()) - - -# --- -page = MainPage("وب:ملعب", "ar") -pageen = MainPage("User:Mr. Ibrahem/sandbox", 'simple') -# --- -existsen = pageen.exists() - -if not page.can_edit(): - print("page can't edit!") -# --- -""" -if page.isRedirect() : return -# target = page.get_redirect_target() -# --- -text = page.get_text() -ns = page.namespace() -links = page.page_links() -categories = page.get_categories(with_hidden=False) -langlinks = page.get_langlinks() -back_links = page.page_backlinks() -wiki_links = page.get_wiki_links_from_text() -words = page.get_words() -templates = page.get_templates() -save_page = page.save(newtext='', summary='', nocreate=1, minor='') -create = page.Create(text='', summary='') -# --- -text_html = page.get_text_html() -hidden_categories= page.get_hidden_categories() -flagged = page.is_flagged() -timestamp = page.get_timestamp() -user = page.get_user() -purge = page.purge() -""" - -# --- -text = page.get_text() -print("---------------------------") -print(f"text:{len(text)=}") -# --- -ex = page.get_wiki_links_from_text() -print("---------------------------") -print(f"get_wiki_links_from_text:{len(ex)=}") -# --- -hidden_categories = page.get_hidden_categories() -print("---------------------------") -print(f"hidden_categories:{len(hidden_categories)=}") -# --- -backlinks = page.page_backlinks() -print("---------------------------") -print(f"backlinks:{len(backlinks)=}") -# --- - -newtext = "تجربة!\n" * 6 -# --- -save = page.save(newtext=newtext) - - -pageen.save(newtext="!!!", nocreate=0) - - -save = page.save(newtext="!") - diff --git a/z_te_sts/test_runner.py b/z_te_sts/test_runner.py deleted file mode 100644 index 5feb561..0000000 --- a/z_te_sts/test_runner.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -""" -Test runner for all z_te_sts/*.py files -Usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" - -import io -from contextlib import redirect_stdout -import sys -import os -import importlib.util -import traceback -from pathlib import Path - -# Add current directory to path for imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - - -def run_test_file(file_path): - """Run a single test file""" - try: - print(f"\n{'='*60}") - print(f"Running: {file_path.name}") - print(f"{'='*60}") - - # Load the module - spec = importlib.util.spec_from_file_location(file_path.stem, file_path) - module = importlib.util.module_from_spec(spec) - - # Capture stdout - - f = io.StringIO() - with redirect_stdout(f): - spec.loader.exec_module(module) - - output = f.getvalue() - if output: - print("Output:") - print(output) - - print(f"✓ {file_path.name} completed successfully") - return True - - except Exception as e: - print(f"✗ {file_path.name} failed: {e}") - print(traceback.format_exc()) - return False - - -def main(): - """Run all test files""" - # Get directory of this script - test_dir = Path(__file__).parent - - # Find all test files - test_files = [] - for file_path in test_dir.glob("test_*.py"): - if file_path.name != "test_runner.py": # Skip this file - test_files.append(file_path) - - if not test_files: - print("No test files found in z_te_sts directory") - return - - # Sort files for consistent execution - test_files.sort() - - print(f"Found {len(test_files)} test files:") - for file_path in test_files: - print(f" - {file_path.name}") - - print("\nStarting test execution...") - - # Run all tests - passed = 0 - failed = 0 - - for file_path in test_files: - if run_test_file(file_path): - passed += 1 - else: - failed += 1 - - # Summary - print(f"\n{'='*60}") - print("Test Summary") - print(f"{'='*60}") - print(f"Total tests: {len(test_files)}") - print(f"Passed: {passed}") - print(f"Failed: {failed}") - - if failed > 0: - print(f"\n❌ {failed} test(s) failed") - sys.exit(1) - else: - print("\n✅ All tests passed!") - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/z_te_sts/test_wiki_page.py b/z_te_sts/test_wiki_page.py deleted file mode 100644 index 8e2ffad..0000000 --- a/z_te_sts/test_wiki_page.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.page import MainPage, CatDepth - -""" -page = MainPage(title, 'ar', family='wikipedia') -exists = page.exists() -text = page.get_text() -timestamp = page.get_timestamp() -user = page.get_user() -links = page.page_links() -words = page.get_words() -purge = page.purge() -templates = page.get_templates() -save_page = page.save(newtext='', summary='', nocreate=1, minor='') -create = page.Create(text='', summary='') -""" -# --- -page = MainPage("تصنيف:اليمن", "ar", family="wikipedia") -# --- -text = page.get_text() -print(f"{len(text)=}") - -# --- -# ex = page.page_backlinks() -# print('---------------------------') -# print(f'page_backlinks:{ex}') -page2 = MainPage("Category:Yemen", "en", family="wikipedia") -# --- -text2 = page2.get_text() -print(f"{len(text2)=}") -# --- -page_backlinks = page.page_backlinks() -print("---------------------------") -print(f"{len(page_backlinks)=}") - -# --- -# --- -# hidden_categories= page.get_hidden_categories() -# print('---------------------------') -# print(f'hidden_categories:{hidden_categories}') -# --- -cat_members = CatDepth("Association football players by nationality", sitecode="en", family="wikipedia", depth=0, ns="14") -# --- -print(f"{len(cat_members)=}") -# --- -red = page.page_links() -print(f"{len(red)=}") -# --- -# save = page.save(newtext='') -# api_new = NEW_API('en', family='wikipedia') -# login = api_new.Login_to_wiki() -# pages = api_new.Find_pages_exists_or_not(liste) -# pages = api_new.Get_Newpages() - - -# python3 core8/pwb.py newapi/wiki_page From 3aacf06dfed859994bbaabd0236e76f3723de1ea Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:07:34 +0300 Subject: [PATCH 05/11] Remove deprecated wiki page modules Deleted user_account_new.py, wiki_page.py, pages_bots/wiki_page.py, and pages_bots_backup/mdwiki_page.py from the newapi package. These modules are no longer needed and have been removed to clean up the codebase. --- newapi/accounts/user_account_new.py | 49 ----------- newapi/pages_bots/wiki_page.py | 94 --------------------- newapi/pages_bots_backup/mdwiki_page.py | 106 ------------------------ newapi/wiki_page.py | 21 ----- 4 files changed, 270 deletions(-) delete mode 100644 newapi/accounts/user_account_new.py delete mode 100644 newapi/pages_bots/wiki_page.py delete mode 100644 newapi/pages_bots_backup/mdwiki_page.py delete mode 100644 newapi/wiki_page.py diff --git a/newapi/accounts/user_account_new.py b/newapi/accounts/user_account_new.py deleted file mode 100644 index 999068f..0000000 --- a/newapi/accounts/user_account_new.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -# --- -from . import user_account_new -# --- -username = user_account_new.bot_username #user_account_new.my_username -password = user_account_new.bot_password #user_account_new.mdwiki_pass -lgpass_enwiki = user_account_new.lgpass_enwiki -user_agent = user_account_new.user_agent -# --- -""" - -import os -import configparser - -home_dir = os.getenv("HOME") -project = home_dir if home_dir else "I:/mdwiki/mdwiki" -# --- -config = configparser.ConfigParser() -config.read(f"{project}/confs/user.ini") - -username = config["DEFAULT"].get("botusername", "") -password = config["DEFAULT"].get("botpassword", "") - -bot_username = config["DEFAULT"].get("botusername", "") -bot_password = config["DEFAULT"].get("botpassword", "") - -my_username = config["DEFAULT"].get("my_username", "") - -mdwiki_pass = config["DEFAULT"].get("mdwiki_pass", "") - -lgpass_enwiki = config["DEFAULT"].get("lgpass_enwiki", "") -my_password = lgpass_enwiki - -qs_token = config["DEFAULT"].get("qs_token", "") - -user_agent = config["DEFAULT"].get("user_agent", "") - -User_tables = { - "username": my_username, - "password": mdwiki_pass, -} - -User_tables_wiki = { - "username": bot_username, - "password": bot_password, -} - -SITECODE = "www" -FAMILY = "mdwiki" diff --git a/newapi/pages_bots/wiki_page.py b/newapi/pages_bots/wiki_page.py deleted file mode 100644 index 8cf6371..0000000 --- a/newapi/pages_bots/wiki_page.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Usage: - -from newapi.page import CatDepth - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], without_lang="", with_lang="", tempyes=[]) - -from newapi.page import MainPage, NEW_API -# api_new = NEW_API('en', family='wikipedia') -# json1 = api_new.post_params(params, addtoken=False) -# move_it = api_new.move(old_title, to, reason="", noredirect=False, movesubpages=False) -# pages = api_new.Find_pages_exists_or_not(liste, get_redirect=False) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# extlinks = api_new.get_extlinks(title) -# revisions= api_new.get_revisions(title) -# logs = api_new.get_logs(title) -# wantedcats = api_new.querypage_list(qppage='Wantedcategories', qplimit="max", Max=5000) -# pages = api_new.Get_template_pages(title, namespace="*", Max=10000) -""" -# --- -import os -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new - -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes - -from ..accounts.user_account_new import User_tables_wiki - -home_dir = os.getenv("HOME") -# --- -User_tables = User_tables_wiki -# --- -print(f"page.py use {User_tables['username']} account.") -# --- -user_agent = default_user_agent() -# --- -change_codes = lang_codes.change_codes - -logins_cache = {} - - -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - - -def MainPage(title, lang, family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode="", family="wikipedia", **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result - - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots_backup/mdwiki_page.py b/newapi/pages_bots_backup/mdwiki_page.py deleted file mode 100644 index 5ae5b36..0000000 --- a/newapi/pages_bots_backup/mdwiki_page.py +++ /dev/null @@ -1,106 +0,0 @@ -# --- -""" - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], onlyns=False, without_lang="", with_lang="", tempyes=[]) - -''' -page = MainPage(title, 'www', family='mdwiki') -exists = page.exists() -if not exists: return -# --- -page_edit = page.can_edit() -if not page_edit: return -# --- -if page.isRedirect() : return -# target = page.get_redirect_target() -# --- -text = page.get_text() -ns = page.namespace() -links = page.page_links() -categories = page.get_categories(with_hidden=False) -langlinks = page.get_langlinks() -wiki_links = page.get_wiki_links_from_text() -refs = page.Get_tags(tag='ref')# for x in ref: name, contents = x.name, x.contents -words = page.get_words() -templates = page.get_templates() -save_page = page.save(newtext='', summary='', nocreate=1, minor='') -create = page.Create(text='', summary='') -# --- -back_links = page.page_backlinks() -text_html = page.get_text_html() -hidden_categories= page.get_hidden_categories() -flagged = page.is_flagged() -timestamp = page.get_timestamp() -user = page.get_user() -purge = page.purge() -''' -""" -# --- -import os -import sys -# --- -home_dir = os.getenv("HOME") -# --- -if "mwclient" not in sys.argv: - sys.argv.append("nomwclient") - # print("sys.argv.append('nomwclient')") - -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes - -from ..accounts.user_account_new import User_tables, SITECODE, FAMILY -# --- -user_agent = default_user_agent() -# --- -change_codes = lang_codes.change_codes - -logins_cache = {} - - -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - - -def MainPage(title, lang, family=FAMILY): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode=SITECODE, family=FAMILY, **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia"): - login_bot = log_it(lang, family) - return bot_api.NEW_API(login_bot, lang=lang, family=family) - - -md_MainPage = MainPage - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'md_MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/wiki_page.py b/newapi/wiki_page.py deleted file mode 100644 index 7acd534..0000000 --- a/newapi/wiki_page.py +++ /dev/null @@ -1,21 +0,0 @@ -""" - - -""" -from .pages_bots.wiki_page import ( - home_dir, - MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes -) - -__all__ = [ - "home_dir", - "user_agent", - "MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] From 5761d57d72f8275a2cacb7085c535dc6e8cf20de Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:11:20 +0300 Subject: [PATCH 06/11] Refactor and remove redundant wiki page modules Deleted new_wiki_pages.py, pages_bots/new_wiki_pages.py, and tt.py as part of a cleanup. Consolidated and refactored related logic into pages_bots/page.py, simplifying API access and removing duplicate code. --- newapi/new_wiki_pages.py | 20 ------- newapi/pages_bots/new_wiki_pages.py | 92 ----------------------------- newapi/pages_bots/page.py | 68 ++++++--------------- newapi/tt.py | 25 -------- 4 files changed, 18 insertions(+), 187 deletions(-) delete mode 100644 newapi/new_wiki_pages.py delete mode 100644 newapi/pages_bots/new_wiki_pages.py delete mode 100644 newapi/tt.py diff --git a/newapi/new_wiki_pages.py b/newapi/new_wiki_pages.py deleted file mode 100644 index ab04eff..0000000 --- a/newapi/new_wiki_pages.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -""" -from .pages_bots.new_wiki_pages import ( - home_dir, - MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes, -) - -__all__ = [ - "home_dir", - "user_agent", - "MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/newapi/pages_bots/new_wiki_pages.py b/newapi/pages_bots/new_wiki_pages.py deleted file mode 100644 index 2f098a0..0000000 --- a/newapi/pages_bots/new_wiki_pages.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -from newapi.page import MainPage, NEW_API, CatDepth - -Usage: - -from newapi.page import CatDepth - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], without_lang="", with_lang="", tempyes=[]) - -from newapi.page import MainPage, NEW_API -# api_new = NEW_API('en', family='wikipedia') -# json1 = api_new.post_params(params, addtoken=False) -# move_it = api_new.move(old_title, to, reason="", noredirect=False, movesubpages=False) -# pages = api_new.Find_pages_exists_or_not(liste, get_redirect=False) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# extlinks = api_new.get_extlinks(title) -# revisions= api_new.get_revisions(title) -# logs = api_new.get_logs(title) -# wantedcats = api_new.querypage_list(qppage='Wantedcategories', qplimit="max", Max=5000) -# pages = api_new.Get_template_pages(title, namespace="*", Max=10000) -""" -# --- -import functools -import os -import sys -from ..super.S_API import bot_api -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes -from .all_apis import ALL_APIS - -from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem -home_dir = os.getenv("HOME") -# --- -User_tables = User_tables_bot -# --- -if "workibrahem" in sys.argv: - User_tables = User_tables_ibrahem - # --- - print(f"page.py use {User_tables['username']} account.") - -user_agent = default_user_agent() -change_codes = lang_codes.change_codes -logins_cache = {} - - -@functools.lru_cache(maxsize=1) -def load_main_api(lang, family) -> ALL_APIS: - return ALL_APIS( - lang=lang, - family=family, - username=User_tables["username"], - password=User_tables["password"], - ) - - -def MainPage(title, lang, family="wikipedia"): - # --- - main_bot = load_main_api(lang, family) - # --- - page = main_bot.MainPage(title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode="", family="wikipedia", **kwargs): - # --- - main_bot = load_main_api(sitecode, family) - # --- - result = main_bot.CatDepth(title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia") -> bot_api.NEW_API: - main_bot = load_main_api(lang, family) - return main_bot.NEW_API() - - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots/page.py b/newapi/pages_bots/page.py index 7155955..ca2a3c8 100644 --- a/newapi/pages_bots/page.py +++ b/newapi/pages_bots/page.py @@ -1,40 +1,13 @@ """ -from newapi.page import MainPage, NEW_API, CatDepth - -Usage: - -from newapi.page import CatDepth - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], without_lang="", with_lang="", tempyes=[]) - -from newapi.page import MainPage, NEW_API -# api_new = NEW_API('en', family='wikipedia') -# json1 = api_new.post_params(params, addtoken=False) -# move_it = api_new.move(old_title, to, reason="", noredirect=False, movesubpages=False) -# pages = api_new.Find_pages_exists_or_not(liste, get_redirect=False) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# extlinks = api_new.get_extlinks(title) -# revisions= api_new.get_revisions(title) -# logs = api_new.get_logs(title) -# wantedcats = api_new.querypage_list(qppage='Wantedcategories', qplimit="max", Max=5000) -# pages = api_new.Get_template_pages(title, namespace="*", Max=10000) """ # --- +import functools import os import sys from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new - -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap from ..api_utils.user_agent import default_user_agent from ..api_utils import lang_codes +from .all_apis import ALL_APIS from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem home_dir = os.getenv("HOME") @@ -45,48 +18,43 @@ User_tables = User_tables_ibrahem # --- print(f"page.py use {User_tables['username']} account.") -# --- + user_agent = default_user_agent() -# --- change_codes = lang_codes.change_codes - logins_cache = {} -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot +@functools.lru_cache(maxsize=1) +def load_main_api(lang, family) -> ALL_APIS: + return ALL_APIS( + lang=lang, + family=family, + username=User_tables["username"], + password=User_tables["password"], + ) def MainPage(title, lang, family="wikipedia"): # --- - login_bot = log_it(lang, family) + main_bot = load_main_api(lang, family) # --- - page = super_page.MainPage(login_bot, title, lang, family=family) + page = main_bot.MainPage(title, lang, family=family) # --- return page def CatDepth(title, sitecode="", family="wikipedia", **kwargs): # --- - login_bot = log_it(sitecode, family) + main_bot = load_main_api(sitecode, family) # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) + result = main_bot.CatDepth(title, sitecode=sitecode, family=family, **kwargs) # --- return result -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result +def NEW_API(lang="", family="wikipedia") -> bot_api.NEW_API: + main_bot = load_main_api(lang, family) + return main_bot.NEW_API() __all__ = [ diff --git a/newapi/tt.py b/newapi/tt.py deleted file mode 100644 index 95677e0..0000000 --- a/newapi/tt.py +++ /dev/null @@ -1,25 +0,0 @@ -# -import os -from configparser import ConfigParser - -if os.getenv("HOME"): - project = "/data/project/himo" -else: - project = 'I:/core/bots/core1' - -db_connect_file = f"{project}/confs/db.ini" - -print(f"{db_connect_file=}") - -main_dir = os.path.dirname(__file__) -print(f"{main_dir=}") -print(f"{__file__=}") - -config2 = ConfigParser() -config2.read(db_connect_file) - -try: - user = config2["client"]["user"] - print(f"{user=}") -except KeyError as e: - print(f"error: {e}") From 4ee0b11a86009f5c2bfba2a0d6c5fa44455d3054 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:13:17 +0300 Subject: [PATCH 07/11] Create pyproject.toml --- pyproject.toml | 147 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cb836ea --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,147 @@ + +# ============================================ +# BLACK +# ============================================ +[tool.black] +line-length = 120 +target-version = ["py313"] +include = '\.pyi?$' +exclude = ''' + /( + \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ +''' + +# ============================================ +# ISORT +# ============================================ +[tool.isort] +profile = "black" +line_length = 120 + +# Keep vertical formatting (multi-line import style) +multi_line_output = 3 +use_parentheses = true +force_grid_wrap = 0 +include_trailing_comma = true + +# OPTIONAL: You can add this to ensure isort does NOT collapse into one line +ensure_newline_before_comments = true + +# Project paths +src_paths = "ArWikiCats" +known_first_party = "ArWikiCats" +skip = [ + ".env", + "env", + ".venv", + "venv", + "build", + "dist", + "__pycache__", + ".mypy_cache", + ".pytest_cache", + ".git", +] + +# ============================================ +# RUFF +# ============================================ +[tool.ruff] +exclude = [ + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "old", + "build", + "dist", + "site-packages", + "venv", +] +line-length = 120 +target-version = "py313" +fix = true + +[tool.ruff.format] +# Allow Ruff/Black to maintain vertical list formatting +skip-magic-trailing-comma = false + +# Keep Black-like behavior +quote-style = "double" +indent-style = "space" +line-ending = "auto" + +# Format code blocks inside docstrings (excellent for README-like docstrings) +docstring-code-format = true +docstring-code-line-length = 120 + +[tool.ruff.lint] +# "E402", "E225", "E226", "E227", "E228", "E252", "F841", "E224", "E203", "F401", +ignore = ["E501", "UP035", "UP006", "N806", "N999", "UP015", "I001", "N802", "UP007", "UP045"] +select = [ + "E", # pycodestyle (error) + "F", # pyflakes + "W", # pycodestyle (warning) + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "PIE", # flake8-pie + "PLE", # pylint error + "RUF100", # Unused noqa comments + "PGH004", # blanket noqa comments + "UP", # pyupgrade + "C4", # flake8-comprehensions + "SIM101", # merge duplicate isinstance calls + "SIM201", "SIM202", "SIM222", "SIM223", # flake8-simplify + "FURB168", # Prefer is operator over isinstance for None checks + "FURB169", # Do not use is comparison with type(None). Use None + "FURB187", # avoid list reverse copy + "FURB188", # use str.remove(pre|suf)fix + "ISC001", # implicitly concatenated string + "RET501", "RET502", # better return None handling +] +# ============================================ +# Flynt +# ============================================ +[tool.flynt] +# Keep it aligned with Black/Ruff +line_length = 120 + +[tool.mypy] +python_version = "3.13" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true +line-length = 120 + +[tool.pylint.messages_control] +max-line-length = 120 +disable = [ + "C0111", # missing-docstring + "C0103", # invalid-name + "R0913", # too-many-arguments +] From cfe9baab024d4383c5d53941b6e1ec307a111f24 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:13:39 +0300 Subject: [PATCH 08/11] Refactor and reformat code for consistency This commit applies consistent formatting, improves import ordering, and enhances code readability across multiple modules. It also includes minor refactoring such as argument alignment, string normalization, and dictionary formatting, but does not introduce new features or change existing logic. --- newapi/DB_bots/db_bot.py | 1 + newapi/DB_bots/pymysql_bot.py | 7 +- newapi/__init__.py | 9 +- newapi/accounts/useraccount.py | 35 +- newapi/all_apis.py | 4 +- newapi/api_utils/ask_bot.py | 3 + newapi/api_utils/botEdit.py | 4 +- .../bot_edit/bot_edit_by_templates.py | 6 +- newapi/api_utils/bot_edit/bot_edit_by_time.py | 10 +- newapi/api_utils/except_err.py | 1 + newapi/api_utils/printe.py | 24 +- newapi/api_utils/txtlib.py | 29 +- newapi/api_utils/user_agent.py | 2 +- newapi/api_utils/wd_sparql.py | 4 +- newapi/except_err.py | 3 +- newapi/page.py | 9 +- newapi/pages_bots/__init__.py | 1 + newapi/pages_bots/all_apis.py | 14 +- newapi/pages_bots/page.py | 23 +- newapi/pformat.py | 19 +- newapi/printe.py | 9 +- newapi/super/Login_db/__init__.py | 2 +- newapi/super/Login_db/bot.py | 26 +- newapi/super/Login_db/bot1.py | 20 +- newapi/super/Login_db/lite.py | 25 +- newapi/super/S_API/bot.py | 52 +- newapi/super/S_API/bot_api.py | 94 +- newapi/super/S_Category/bot.py | 14 +- newapi/super/S_Category/catdepth_new.py | 15 +- newapi/super/S_Page/bot.py | 15 +- newapi/super/S_Page/data.py | 15 + newapi/super/S_Page/super_page.py | 65 +- newapi/super/__init__.py | 18 +- newapi/super/bot.py | 32 +- newapi/super/bot_new.py | 30 +- newapi/super/cookies_bot.py | 8 +- newapi/super/handel_errors.py | 13 +- newapi/super/login_wrap.py | 7 +- newapi/super/login_wrap_new.py | 6 +- newapi/super/mwclient/__init__.py | 45 +- newapi/super/mwclient/client.py | 1036 ++++++++++------- newapi/super/mwclient/errors.py | 32 +- newapi/super/mwclient/image.py | 68 +- newapi/super/mwclient/listing.py | 157 ++- newapi/super/mwclient/page.py | 394 ++++--- newapi/super/mwclient/sleep.py | 7 +- newapi/super/mwclient/util.py | 10 +- newapi/super/params_help.py | 18 +- newapi/super/super_login.py | 19 +- tests/TestALL_APIS.py | 61 +- tests/TestAuthentication.py | 17 +- tests/TestLiteDB.py | 8 +- tests/TestMainPage.py | 7 +- tests/TestNewAPI.py | 5 +- tests/conftest.py | 15 +- .../test_bot_edit_by_templates.py | 15 +- .../test_bot_edit_by_templates2.py | 219 ++-- .../test_bot_edit_by_templates_pypass.py | 22 +- .../bot_edit_by_time/test_bot_edit_by_time.py | 6 +- 59 files changed, 1576 insertions(+), 1259 deletions(-) diff --git a/newapi/DB_bots/db_bot.py b/newapi/DB_bots/db_bot.py index 41330fa..9e15f11 100644 --- a/newapi/DB_bots/db_bot.py +++ b/newapi/DB_bots/db_bot.py @@ -10,6 +10,7 @@ # db.select(table_name, args) """ + # --- import sqlite_utils diff --git a/newapi/DB_bots/pymysql_bot.py b/newapi/DB_bots/pymysql_bot.py index bb6d43c..7be0cc5 100644 --- a/newapi/DB_bots/pymysql_bot.py +++ b/newapi/DB_bots/pymysql_bot.py @@ -2,13 +2,18 @@ from newapi import pymysql_bot # result = pymysql_bot.sql_connect_pymysql(query, return_dict=False, values=None, main_args={}, credentials={}, conversions=None) """ + import copy + import pymysql import pymysql.cursors + from ..api_utils.except_err import exception_err -def sql_connect_pymysql(query, return_dict=False, values=None, main_args={}, credentials={}, conversions=None, many=False, **kwargs): +def sql_connect_pymysql( + query, return_dict=False, values=None, main_args={}, credentials={}, conversions=None, many=False, **kwargs +): args = copy.deepcopy(main_args) args["cursorclass"] = pymysql.cursors.DictCursor if return_dict else pymysql.cursors.Cursor if conversions: diff --git a/newapi/__init__.py b/newapi/__init__.py index e29eeef..48d76c8 100644 --- a/newapi/__init__.py +++ b/newapi/__init__.py @@ -2,13 +2,12 @@ """ from newapi import useraccount """ -from .DB_bots import db_bot, pymysql_bot -from .accounts import useraccount -from .api_utils import except_err, botEdit -from .api_utils import printe, txtlib, wd_sparql -from .super.login_wrap import LoginWrap from . import page +from .accounts import useraccount from .all_apis import ALL_APIS +from .api_utils import botEdit, except_err, printe, txtlib, wd_sparql +from .DB_bots import db_bot, pymysql_bot +from .super.login_wrap import LoginWrap __all__ = [ "ALL_APIS", diff --git a/newapi/accounts/useraccount.py b/newapi/accounts/useraccount.py index a9c9584..b8ec71c 100644 --- a/newapi/accounts/useraccount.py +++ b/newapi/accounts/useraccount.py @@ -3,39 +3,40 @@ from newapi.accounts.useraccount import User_tables_bot, User_tables_ibrahem """ -import sys -import os + import configparser +import os +import sys -project = '/data/project/himo' +project = "/data/project/himo" # --- if not os.path.isdir(project): - project = 'I:/core/bots/core1' + project = "I:/core/bots/core1" # --- config = configparser.ConfigParser() config.read(f"{project}/confs/user.ini") -DEFAULT = config['DEFAULT'] +DEFAULT = config["DEFAULT"] -username = config['DEFAULT'].get('botusername', "") -password = config['DEFAULT'].get('botpassword', "") +username = config["DEFAULT"].get("botusername", "") +password = config["DEFAULT"].get("botpassword", "") # --- -passworden = config['DEFAULT'].get('passworden', "") -passwordwd = config['DEFAULT'].get('passwordwd', "") +passworden = config["DEFAULT"].get("passworden", "") +passwordwd = config["DEFAULT"].get("passwordwd", "") -password_ar = config['DEFAULT'].get('password_ar', "") -password_en = config['DEFAULT'].get('password_en', "") +password_ar = config["DEFAULT"].get("password_ar", "") +password_en = config["DEFAULT"].get("password_en", "") -hiacc = config['DEFAULT'].get('hiacc', "") -hipass = config['DEFAULT'].get('hipass', "") +hiacc = config["DEFAULT"].get("hiacc", "") +hipass = config["DEFAULT"].get("hipass", "") -mdwiki_pass = config['DEFAULT'].get('mdwiki_pass', "") +mdwiki_pass = config["DEFAULT"].get("mdwiki_pass", "") -qs_token = config['DEFAULT'].get('qs_token', "") -qs_tokenbot = config['DEFAULT'].get('qs_tokenbot', "") +qs_token = config["DEFAULT"].get("qs_token", "") +qs_tokenbot = config["DEFAULT"].get("qs_tokenbot", "") -user_agent = config['DEFAULT'].get('user_agent', "") +user_agent = config["DEFAULT"].get("user_agent", "") if "workibrahem" in sys.argv: username = hiacc diff --git a/newapi/all_apis.py b/newapi/all_apis.py index e55c0c5..e7e000c 100644 --- a/newapi/all_apis.py +++ b/newapi/all_apis.py @@ -1,5 +1,5 @@ -""" -""" +""" """ + from .pages_bots.all_apis import ( ALL_APIS, ) diff --git a/newapi/api_utils/ask_bot.py b/newapi/api_utils/ask_bot.py index 685c49f..3a99894 100644 --- a/newapi/api_utils/ask_bot.py +++ b/newapi/api_utils/ask_bot.py @@ -3,13 +3,16 @@ from ...api_utils.ask_bot import ASK_BOT """ + import sys + from . import printe yes_answer = ["y", "a", "", "Y", "A", "all", "aaa"] Save_or_Ask = {} + class ASK_BOT: def __init__(self): pass diff --git a/newapi/api_utils/botEdit.py b/newapi/api_utils/botEdit.py index 86806ba..1a9b768 100644 --- a/newapi/api_utils/botEdit.py +++ b/newapi/api_utils/botEdit.py @@ -2,10 +2,12 @@ from newapi.api_utils import botEdit bot_edit! """ + +from .bot_edit.bot_edit_by_templates import is_bot_edit_allowed + # # from .bot_edit.bot_edit_by_time import check_create_time, check_last_edit_time -from .bot_edit.bot_edit_by_templates import is_bot_edit_allowed Created_Cache = {} diff --git a/newapi/api_utils/bot_edit/bot_edit_by_templates.py b/newapi/api_utils/bot_edit/bot_edit_by_templates.py index 2aa7f8b..0abb584 100644 --- a/newapi/api_utils/bot_edit/bot_edit_by_templates.py +++ b/newapi/api_utils/bot_edit/bot_edit_by_templates.py @@ -2,10 +2,12 @@ from newapi.api_utils import botEdit bot_edit! """ + # # import logging import sys + import wikitextparser as wtp logger = logging.getLogger(__name__) @@ -126,7 +128,7 @@ def is_bot_edit_allowed(text="", title_page="", botjob="all"): title = str(template.normal_name()).strip() # --- params = { - str(param.name).strip() : str(param.value).strip() + str(param.name).strip(): str(param.value).strip() for param in template.arguments if str(param.value).strip() } @@ -140,7 +142,7 @@ def is_bot_edit_allowed(text="", title_page="", botjob="all"): Bot_Cache[botjob][title_page] = False return False # --- - logger.debug('<>botEdit.py title:(%s), params:(%s).' % (title, str(params))) + logger.debug("<>botEdit.py title:(%s), params:(%s)." % (title, str(params))) # --- if title.lower() == "nobots": return _handle_nobots_template(params, title_page, botjob, _template) diff --git a/newapi/api_utils/bot_edit/bot_edit_by_time.py b/newapi/api_utils/bot_edit/bot_edit_by_time.py index 85578c5..6bce11a 100644 --- a/newapi/api_utils/bot_edit/bot_edit_by_time.py +++ b/newapi/api_utils/bot_edit/bot_edit_by_time.py @@ -1,7 +1,9 @@ -""" -""" +""" """ + import datetime + from .. import printe + Created_Cache = {} @@ -80,7 +82,9 @@ def check_last_edit_time(page, title_page, delay): # --- if diff_minutes < delay: printe.output(f"<>Page:{title_page} last edit ({timestamp}).") - printe.output(f"<>Page Last edit before {delay} minutes, Wait {wait_time:.2f} minutes. title:{title_page}") + printe.output( + f"<>Page Last edit before {delay} minutes, Wait {wait_time:.2f} minutes. title:{title_page}" + ) return False # --- return True diff --git a/newapi/api_utils/except_err.py b/newapi/api_utils/except_err.py index 98c26aa..24784bc 100644 --- a/newapi/api_utils/except_err.py +++ b/newapi/api_utils/except_err.py @@ -6,6 +6,7 @@ from .except_err import exception_err, warn_err """ + import inspect import traceback from warnings import warn diff --git a/newapi/api_utils/printe.py b/newapi/api_utils/printe.py index 8be1b76..ebf1d75 100644 --- a/newapi/api_utils/printe.py +++ b/newapi/api_utils/printe.py @@ -9,18 +9,17 @@ printe.showDiff('old text', 'new text') # prints the differences between 'old text' and 'new text' """ +import difflib + # --- import functools -import difflib +import logging import re import sys - from collections import abc +from collections.abc import Iterable, Sequence from difflib import _format_range_unified as format_range_unified from itertools import zip_longest -from collections.abc import Iterable, Sequence - -import logging if "debug" in sys.argv: logging.basicConfig(level=logging.DEBUG) @@ -230,7 +229,7 @@ def get_color_table(): "lightblack": 108, "bold": 1, } - color_table = {x : f"\033[{v}m%s\033[00m" for x, v in color_numbers.items()} + color_table = {x: f"\033[{v}m%s\033[00m" for x, v in color_numbers.items()} # Add light versions of the colors to the color table for color in ["purple", "yellow", "blue", "red", "green", "cyan", "gray"]: @@ -344,7 +343,9 @@ class Hunk: NOT_APPR = -1 PENDING = 0 - def __init__(self, a: str | Sequence[str], b: str | Sequence[str], grouped_opcode: Sequence[tuple[str, int, int, int, int]]) -> None: + def __init__( + self, a: str | Sequence[str], b: str | Sequence[str], grouped_opcode: Sequence[tuple[str, int, int, int, int]] + ) -> None: """ Initializer. @@ -537,7 +538,9 @@ def get_header_text(a_rng: tuple[int, int], b_rng: tuple[int, int], affix: str = class PatchManager: - def __init__(self, text_a: str, text_b: str, context: int = 0, by_letter: bool = False, replace_invisible: bool = False) -> None: + def __init__( + self, text_a: str, text_b: str, context: int = 0, by_letter: bool = False, replace_invisible: bool = False + ) -> None: self.a = text_a.splitlines(True) self.b = text_b.splitlines(True) @@ -629,7 +632,10 @@ def _get_context_range(self, super_hunk: _Superhunk) -> tuple[tuple[int, int], t """Dynamically determine context range for a super hunk.""" a0, a1 = super_hunk.a_rng b0, b1 = super_hunk.b_rng - return ((a0 - min(super_hunk.pre_context, self.context), a1 + min(super_hunk.post_context, self.context)), (b0 - min(super_hunk.pre_context, self.context), b1 + min(super_hunk.post_context, self.context))) + return ( + (a0 - min(super_hunk.pre_context, self.context), a1 + min(super_hunk.post_context, self.context)), + (b0 - min(super_hunk.pre_context, self.context), b1 + min(super_hunk.post_context, self.context)), + ) def _generate_diff(self, hunks: _Superhunk) -> str: """Generate a diff text for the given hunks.""" diff --git a/newapi/api_utils/txtlib.py b/newapi/api_utils/txtlib.py index 802c8bb..dd6559d 100644 --- a/newapi/api_utils/txtlib.py +++ b/newapi/api_utils/txtlib.py @@ -13,9 +13,10 @@ """ # from ..other_bots import printe -import wikitextparser as wtp from functools import lru_cache +import wikitextparser as wtp + @lru_cache(maxsize=512) def extract_templates_and_params(text): @@ -24,7 +25,7 @@ def extract_templates_and_params(text): # --- parsed = wtp.parse(text) templates = parsed.templates - arguments = 'arguments' + arguments = "arguments" # --- for template in templates: # --- @@ -46,10 +47,10 @@ def extract_templates_and_params(text): namestrip = name # --- ficrt = { - 'name': f"قالب:{name}", - 'namestrip': namestrip, - 'params': params, - 'item': pa_item, + "name": f"قالب:{name}", + "namestrip": namestrip, + "params": params, + "item": pa_item, } # --- result.append(ficrt) @@ -65,7 +66,7 @@ def get_one_temp_params(text, tempname="", templates=[], lowers=False, get_all_t if tempname: temps.append(tempname) # --- - temps = [x.replace("قالب:", "").replace("Template:", "").replace('_', ' ').strip() for x in temps] + temps = [x.replace("قالب:", "").replace("Template:", "").replace("_", " ").strip() for x in temps] # --- if lowers: temps = [x.lower() for x in temps] @@ -78,7 +79,7 @@ def get_one_temp_params(text, tempname="", templates=[], lowers=False, get_all_t for temp in ingr: # --- # name, namestrip, params, template = temp['name'], temp['namestrip'], temp['params'], temp['item'] - namestrip, params = temp['namestrip'], temp['params'] + namestrip, params = temp["namestrip"], temp["params"] # --- if lowers: namestrip = namestrip.lower() @@ -89,9 +90,7 @@ def get_one_temp_params(text, tempname="", templates=[], lowers=False, get_all_t # --- # print("te:%s, namestrip:%s" % (te,namestrip) ) # --- - tabe = { - namestrip: params - } + tabe = {namestrip: params} named.append(tabe) # --- return named @@ -108,7 +107,7 @@ def get_all_temps_params(text, templates=None, lowers=False): # --- -test_text = ''' +test_text = """ {{ص.م/صورة مضاعفة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}} | صورة1 ={{{علم|{{{flag|{{{صورة علم|}}}}}}}}} | تعليق1 ={{#لو:{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{وصف العلم|{{{flagcaption|}}}}}}|خاصية=P163|rank=best}}|{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{وصف العلم|{{{flagcaption|}}}}}}|خاصية=P163|rank=best}}|{{فصع}}}} @@ -122,15 +121,15 @@ def get_all_temps_params(text, templates=None, lowers=False): }} {{ourworldindatamirror|https://owidm.wmcloud.org/grapher/cancer-death-rates?tab=map {{Webarchive}}}} -''' +""" # --- -if __name__ == '__main__': +if __name__ == "__main__": # --- # --- ingr = extract_templates_and_params(test_text) for temp in ingr: # --- - name, namestrip, params, template = temp['name'], temp['namestrip'], temp['params'], temp['item'] + name, namestrip, params, template = temp["name"], temp["namestrip"], temp["params"], temp["item"] # --- print("-----------------------------") print(f"name: {name}") diff --git a/newapi/api_utils/user_agent.py b/newapi/api_utils/user_agent.py index c4c83a1..62b9a0b 100644 --- a/newapi/api_utils/user_agent.py +++ b/newapi/api_utils/user_agent.py @@ -3,6 +3,7 @@ from .api_utils.user_agent import default_user_agent """ + import os from functools import lru_cache @@ -18,4 +19,3 @@ def default_user_agent(): # printe.output(f"default_user_agent: {li}") # --- return li - diff --git a/newapi/api_utils/wd_sparql.py b/newapi/api_utils/wd_sparql.py index 61ae757..f46c452 100644 --- a/newapi/api_utils/wd_sparql.py +++ b/newapi/api_utils/wd_sparql.py @@ -6,8 +6,10 @@ get_query_data = wd_sparql.get_query_data """ + import sys -from SPARQLWrapper import SPARQLWrapper, JSON + +from SPARQLWrapper import JSON, SPARQLWrapper def get_query_data(query): diff --git a/newapi/except_err.py b/newapi/except_err.py index 8b1046f..a812440 100644 --- a/newapi/except_err.py +++ b/newapi/except_err.py @@ -1,6 +1,5 @@ -""" +""" """ -""" from .api_utils.except_err import exception_err, warn_err __all__ = [ diff --git a/newapi/page.py b/newapi/page.py index 026a690..51d7cad 100644 --- a/newapi/page.py +++ b/newapi/page.py @@ -1,13 +1,12 @@ -""" +""" """ -""" from .pages_bots.page import ( - home_dir, - MainPage, - user_agent, NEW_API, CatDepth, + MainPage, change_codes, + home_dir, + user_agent, ) __all__ = [ diff --git a/newapi/pages_bots/__init__.py b/newapi/pages_bots/__init__.py index d0add0e..6017056 100644 --- a/newapi/pages_bots/__init__.py +++ b/newapi/pages_bots/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os + if not os.getenv("BOTNAME"): botname = "newapi" os.environ["BOTNAME"] = botname diff --git a/newapi/pages_bots/all_apis.py b/newapi/pages_bots/all_apis.py index 86ba868..4e690dc 100644 --- a/newapi/pages_bots/all_apis.py +++ b/newapi/pages_bots/all_apis.py @@ -8,13 +8,15 @@ cat_members = main_api.CatDepth('Category Title') new_api = main_api.NEW_API() """ + # --- import functools from typing import Any + +from ..api_utils import printe from ..super.S_API import bot_api from ..super.S_Category import catdepth_new from ..super.S_Page import super_page -from ..api_utils import printe from ..super.super_login import Login @@ -52,12 +54,14 @@ def _login(self) -> Login: # --- login_bot = Login(self.lang, family=self.family) # --- - printe.output(f"### <> LoginWrap make new bot for ({self.lang}.{self.family}.org|{self.username})", p=True) + printe.output( + f"### <> LoginWrap make new bot for ({self.lang}.{self.family}.org|{self.username})", p=True + ) # --- user_tables = { self.family: { - 'username': self.username, - 'password': self.password, + "username": self.username, + "password": self.password, } } # --- @@ -67,5 +71,5 @@ def _login(self) -> Login: __all__ = [ - 'ALL_APIS', + "ALL_APIS", ] diff --git a/newapi/pages_bots/page.py b/newapi/pages_bots/page.py index ca2a3c8..23299fe 100644 --- a/newapi/pages_bots/page.py +++ b/newapi/pages_bots/page.py @@ -1,15 +1,16 @@ -""" -""" +""" """ + # --- import functools import os import sys -from ..super.S_API import bot_api -from ..api_utils.user_agent import default_user_agent + +from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem from ..api_utils import lang_codes +from ..api_utils.user_agent import default_user_agent +from ..super.S_API import bot_api from .all_apis import ALL_APIS -from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem home_dir = os.getenv("HOME") # --- User_tables = User_tables_bot @@ -58,10 +59,10 @@ def NEW_API(lang="", family="wikipedia") -> bot_api.NEW_API: __all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', + "home_dir", + "user_agent", + "MainPage", + "NEW_API", + "CatDepth", + "change_codes", ] diff --git a/newapi/pformat.py b/newapi/pformat.py index d29a918..5547640 100644 --- a/newapi/pformat.py +++ b/newapi/pformat.py @@ -1,10 +1,11 @@ """ python3 core8/pwb.py newapi/pformat """ + import sys -import wikitextparser as wtp from pathlib import Path +import wikitextparser as wtp # python3 core8/pwb.py newapi/pformat -title:قالب:Cycling_race/stageclassification3 # python3 core8/pwb.py newapi/pformat -title:قالب:معلومات_خاصية_ويكي_بيانات/تتبع @@ -31,27 +32,27 @@ def make_new_text(text): if __name__ == "__main__": # --- - sys.argv.append('workibrahem') - sys.argv.append('ask') + sys.argv.append("workibrahem") + sys.argv.append("ask") # --- Dir = Path(__file__).parent # --- - title = '' + title = "" text = open(f"{Dir}/pformat.txt", "r", encoding="utf-8").read() for arg in sys.argv: - arg, _, value = arg.partition(':') - if arg == '-title' or arg == '-page': + arg, _, value = arg.partition(":") + if arg == "-title" or arg == "-page": title = value # --- if title: # --- from newapi.page import MainPage - page = MainPage(title, 'ar', family='wikipedia') + page = MainPage(title, "ar", family="wikipedia") text = page.get_text() newtext = make_new_text(text) - if 'save' in sys.argv: - save_page = page.save(newtext=newtext, summary='', nocreate=1, minor='') + if "save" in sys.argv: + save_page = page.save(newtext=newtext, summary="", nocreate=1, minor="") else: prased = wtp.parse(text) newtext = prased.pformat() diff --git a/newapi/printe.py b/newapi/printe.py index 2cf9914..7d75fbc 100644 --- a/newapi/printe.py +++ b/newapi/printe.py @@ -1,14 +1,13 @@ -""" +""" """ -""" from .api_utils.printe import ( - showDiff, - output, debug, - warn, error, info, + output, + showDiff, test_print, + warn, ) __all__ = [ diff --git a/newapi/super/Login_db/__init__.py b/newapi/super/Login_db/__init__.py index 7c68785..40a96af 100644 --- a/newapi/super/Login_db/__init__.py +++ b/newapi/super/Login_db/__init__.py @@ -1 +1 @@ -# -*- coding: utf-8 -*- \ No newline at end of file +# -*- coding: utf-8 -*- diff --git a/newapi/super/Login_db/bot.py b/newapi/super/Login_db/bot.py index e2f0882..83af18f 100644 --- a/newapi/super/Login_db/bot.py +++ b/newapi/super/Login_db/bot.py @@ -6,11 +6,13 @@ # log_one(site="", user="", result="", db_type="sqlite", db_path="login_logs.db", credentials=None) """ + +import datetime import functools -import sys import os -import datetime +import sys from configparser import ConfigParser + from ...api_utils import printe from ...DB_bots.pymysql_bot import sql_connect_pymysql @@ -40,7 +42,7 @@ def jls_extract_def(): # --- project = os.getenv("HOME") # --- - if __file__.find('/data/project/himo') != -1: + if __file__.find("/data/project/himo") != -1: project = "/data/project/himo" # --- db_connect_file = f"{project}/confs/db.ini" @@ -88,11 +90,11 @@ def log_one(site, user, result, action="", params={}) -> None: params = params or {} # --- if action == "query": - if params.get('meta', "") == "tokens": + if params.get("meta", "") == "tokens": action = "tokens" - if params.get('type'): - action += "_" + params['type'] - elif params.get('meta', "").find("userinfo") != -1: + if params.get("type"): + action += "_" + params["type"] + elif params.get("meta", "").find("userinfo") != -1: action = "userinfo" # --- # if params.get('meta', "").find("userinfo") != -1: @@ -100,8 +102,8 @@ def log_one(site, user, result, action="", params={}) -> None: # if userinfo > 3: # raise Exception("too much") # --- - if action == "query" and params.get('prop'): - action += "_" + "_".join(params['prop'].split("|")) + if action == "query" and params.get("prop"): + action += "_" + "_".join(params["prop"].split("|")) # --- if action == "query" and "test" in sys.argv: printe.output(f"<> {action=}") @@ -142,11 +144,7 @@ def log_one(site, user, result, action="", params={}) -> None: timestamp = datetime.datetime.now().isoformat() # Execute table creation - sql_connect_pymysql( - create_table_query, - main_args=main_args, - credentials=credentials - ) + sql_connect_pymysql(create_table_query, main_args=main_args, credentials=credentials) # Insert login attempt insert_query = """ diff --git a/newapi/super/Login_db/bot1.py b/newapi/super/Login_db/bot1.py index e248119..e1cb9e3 100644 --- a/newapi/super/Login_db/bot1.py +++ b/newapi/super/Login_db/bot1.py @@ -6,19 +6,21 @@ # log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=login_result) """ -import sys -import os + +import datetime import json +import os +import sys + import pymysql -import datetime DB_CONFIG = { - 'host': 'localhost', - 'user': 'root', - 'password': 'password', - 'database': 'login_db', - 'charset': 'utf8mb4', - 'cursorclass': pymysql.cursors.DictCursor + "host": "localhost", + "user": "root", + "password": "password", + "database": "login_db", + "charset": "utf8mb4", + "cursorclass": pymysql.cursors.DictCursor, } LOGS_IS_ENABLED = os.getenv("LOGIN_LOGS_IS_ENABLED", "0") == "1" diff --git a/newapi/super/Login_db/lite.py b/newapi/super/Login_db/lite.py index 3c52ea1..c99bd15 100644 --- a/newapi/super/Login_db/lite.py +++ b/newapi/super/Login_db/lite.py @@ -6,12 +6,16 @@ # log_one(site="", user="", result="", db_type="sqlite", db_path="login_logs.db", credentials=None) """ -from ...api_utils.except_err import exception_err + import datetime from pathlib import Path + +from ...api_utils.except_err import exception_err from ...DB_bots import db_bot + LiteDB = db_bot.LiteDB + def log_one_sqlite(site="", user="", result=""): Dir = Path(__file__).resolve().parent @@ -24,24 +28,13 @@ def log_one_sqlite(site="", user="", result=""): # Create table if it doesn't exist db.create_table( - "login_attempts", - { - "id": int, - "site": str, - "username": str, - "result": str, - "timestamp": str - }, - pk="id" + "login_attempts", {"id": int, "site": str, "username": str, "result": str, "timestamp": str}, pk="id" ) # Insert login attempt - db.insert("login_attempts", { - "site": site, - "username": user, - "result": result, - "timestamp": timestamp - }, check=False) # Don't check for duplicates since each attempt should be logged + db.insert( + "login_attempts", {"site": site, "username": user, "result": result, "timestamp": timestamp}, check=False + ) # Don't check for duplicates since each attempt should be logged except Exception as e: exception_err(e, "Failed to log login attempt to SQLite") diff --git a/newapi/super/S_API/bot.py b/newapi/super/S_API/bot.py index 1a17600..35cabb6 100644 --- a/newapi/super/S_API/bot.py +++ b/newapi/super/S_API/bot.py @@ -3,10 +3,12 @@ from .super.S_API.bot import BOTS_APIS """ + import sys + from ...api_utils import printe -from ..handel_errors import HANDEL_ERRORS from ...api_utils.ask_bot import ASK_BOT +from ..handel_errors import HANDEL_ERRORS yes_answer = ["y", "a", "", "Y", "A", "all", "aaa"] file_name = "bot_api.py" @@ -33,9 +35,15 @@ def Add_To_Bottom(self, text, summary, title, poss="Head|Bottom"): printe.test_print(f"** Add_To_Bottom .. [[{title}]] ") # printe.showDiff("", text) # --- - user = self.username or getattr(self, 'user_login', '') + user = self.username or getattr(self, "user_login", "") # --- - ask = self.ask_put(newtext=text, message=f"** Add_To {poss} .. [[{title}]] ", job="Add_To_Bottom", username=user, summary=summary) + ask = self.ask_put( + newtext=text, + message=f"** Add_To {poss} .. [[{title}]] ", + job="Add_To_Bottom", + username=user, + summary=summary, + ) # --- if ask is False: return False @@ -104,7 +112,7 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r # --- message = f"Do you want to move page:[[{old_title}]] to [[{to}]]?" # --- - user = self.username or getattr(self, 'user_login', '') + user = self.username or getattr(self, "user_login", "") # --- if not self.ask_put(message=message, job="move", username=user): return {} @@ -123,9 +131,30 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r "reason": "wrong title", "redirectcreated": True, "moveoverredirect": False, - "talkmove-errors": [{"message": "content-not-allowed-here", "params": ["Structured Discussions board", "User talk:Mr. Ibrahem/x", "main"], "code": "contentnotallowedhere", "type": "error"}, {"message": "flow-error-allowcreation-flow-create-board", "params": [], "code": "flow-error-allowcreation-flow-create-board", "type": "error"}], - "subpages": {"errors": [{"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"}]}, - "subpages-talk": {"errors": [{"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"}]}, + "talkmove-errors": [ + { + "message": "content-not-allowed-here", + "params": ["Structured Discussions board", "User talk:Mr. Ibrahem/x", "main"], + "code": "contentnotallowedhere", + "type": "error", + }, + { + "message": "flow-error-allowcreation-flow-create-board", + "params": [], + "code": "flow-error-allowcreation-flow-create-board", + "type": "error", + }, + ], + "subpages": { + "errors": [ + {"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"} + ] + }, + "subpages-talk": { + "errors": [ + {"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"} + ] + }, } } # --- @@ -146,7 +175,14 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r if error: if error_code == "ratelimited": printe.output("<> move ratelimited:") - return self.move(old_title, to, reason=reason, noredirect=noredirect, movesubpages=movesubpages, return_dict=return_dict) + return self.move( + old_title, + to, + reason=reason, + noredirect=noredirect, + movesubpages=movesubpages, + return_dict=return_dict, + ) if error_code == "articleexists": printe.output("<> articleexists") diff --git a/newapi/super/S_API/bot_api.py b/newapi/super/S_API/bot_api.py index 1d31a3e..b43478c 100644 --- a/newapi/super/S_API/bot_api.py +++ b/newapi/super/S_API/bot_api.py @@ -40,16 +40,19 @@ api_new = NEW_API(code, family='wikipedia') api_new.Login_to_wiki() """ -# --- -import tqdm -import time + +import datetime import sys +import time from collections.abc import KeysView -import datetime from datetime import timedelta + +# --- +import tqdm + from ...api_utils import printe -from .bot import BOTS_APIS from ...api_utils.lang_codes import change_codes +from .bot import BOTS_APIS class NEW_API(BOTS_APIS): @@ -74,25 +77,14 @@ def __init__(self, login_bot, lang="", family="wikipedia"): def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): # --- return self.login_bot.post_params( - params, - Type=Type, - addtoken=addtoken, - GET_CSRF=GET_CSRF, - files=files, - do_error=do_error, - max_retry=max_retry + params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry ) - def post_continue(self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None): + def post_continue( + self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + ): return self.login_bot.post_continue( - params, - action, - _p_=_p_, - p_empty=p_empty, - Max=Max, - first=first, - _p_2=_p_2, - _p_2_empty=_p_2_empty + params, action, _p_=_p_, p_empty=p_empty, Max=Max, first=first, _p_2=_p_2, _p_2_empty=_p_2_empty ) def get_username(self): @@ -140,7 +132,7 @@ def Find_pages_exists_or_not(self, liste, get_redirect=False, noprint=False): query_table = all_jsons.get("query", {}) # --- normalz = query_table.get("normalized", []) - normalized = {red["to"] : red["from"] for red in normalz} + normalized = {red["to"]: red["from"] for red in normalz} # --- query_pages = query_table.get("pages", []) # --- @@ -174,7 +166,15 @@ def Find_pages_exists_or_not(self, liste, get_redirect=False, noprint=False): # --- return table - def Find_pages_exists_or_not_with_qids(self, liste, get_redirect=False, noprint=False, return_all_jsons=False, use_user_input_title=False, chunk_size=50): + def Find_pages_exists_or_not_with_qids( + self, + liste, + get_redirect=False, + noprint=False, + return_all_jsons=False, + use_user_input_title=False, + chunk_size=50, + ): # --- done = 0 # --- @@ -240,10 +240,7 @@ def Find_pages_exists_or_not_with_qids(self, liste, get_redirect=False, noprint= if use_user_input_title and title_tab.get("user_input"): title_x = title_tab["user_input"] # --- - table.setdefault(title_x, { - "wikibase_item": wikibase_item, - "exist": False - }) + table.setdefault(title_x, {"wikibase_item": wikibase_item, "exist": False}) # --- if title_tab: table[title_x]["title_tab"] = title_tab @@ -272,7 +269,9 @@ def Find_pages_exists_or_not_with_qids(self, liste, get_redirect=False, noprint= def Get_All_pages(self, start="", namespace="0", limit="max", apfilterredir="", ppprop="", limit_all=100000): # --- - printe.test_print(f"Get_All_pages for start:{start}, limit:{limit},namespace:{namespace},apfilterredir:{apfilterredir}") + printe.test_print( + f"Get_All_pages for start:{start}, limit:{limit},namespace:{namespace},apfilterredir:{apfilterredir}" + ) # --- params = { "action": "query", @@ -368,9 +367,13 @@ def PrefixSearch(self, pssearch="", ns="0", pslimit="max", limit_all=100000): # --- return Main_table - def Get_All_pages_generator(self, start="", namespace="0", limit="max", filterredir="", ppprop="", limit_all=100000): + def Get_All_pages_generator( + self, start="", namespace="0", limit="max", filterredir="", ppprop="", limit_all=100000 + ): # --- - printe.test_print(f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}") + printe.test_print( + f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}" + ) # --- params = { "action": "query", @@ -449,7 +452,16 @@ def Search(self, value="", ns="*", offset="", srlimit="max", RETURN_dict=False, # --- return results - def Get_Newpages(self, limit=5000, namespace="0", rcstart="", user="", three_houers=False, offset_minutes=False, offset_hours=False): + def Get_Newpages( + self, + limit=5000, + namespace="0", + rcstart="", + user="", + three_houers=False, + offset_minutes=False, + offset_hours=False, + ): if three_houers: dd = datetime.datetime.utcnow() - timedelta(hours=3) rcstart = dd.strftime("%Y-%m-%dT%H:%M:00.000Z") @@ -617,7 +629,9 @@ def Get_langlinks_for_list(self, titles, targtsitecode="", numbes=40): if lang["lang"] == targtsitecode: find_targtsitecode += 1 # --- - printe.output(f'bot_api.Get_langlinks_for_list find "{len(table)}" in table,find_targtsitecode:{targtsitecode}:{find_targtsitecode}') + printe.output( + f'bot_api.Get_langlinks_for_list find "{len(table)}" in table,find_targtsitecode:{targtsitecode}:{find_targtsitecode}' + ) # --- return table @@ -933,10 +947,22 @@ def users_infos(self, ususers=[]): "utf8": 1, "formatversion": "2", "usprop": "groups|implicitgroups|editcount|gender|registration", - "ususers": "Mr.Ibrahembot" + "ususers": "Mr.Ibrahembot", } # --- - all_usprops = ["groups", "implicitgroups", "cancreate", "editcount", "centralids", "blockinfo", "emailable", "gender", "groupmemberships", "registration", "rights"] + all_usprops = [ + "groups", + "implicitgroups", + "cancreate", + "editcount", + "centralids", + "blockinfo", + "emailable", + "gender", + "groupmemberships", + "registration", + "rights", + ] # --- ususers = list(set(ususers)) # --- diff --git a/newapi/super/S_Category/bot.py b/newapi/super/S_Category/bot.py index e3840cd..b31ac9c 100644 --- a/newapi/super/S_Category/bot.py +++ b/newapi/super/S_Category/bot.py @@ -3,7 +3,9 @@ from .bot import CategoryDepth """ + import copy + import tqdm from ...api_utils import printe @@ -67,13 +69,7 @@ def __init__(self, login_bot, title, **kwargs): def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): # --- return self.login_bot.post_params( - params, - Type=Type, - addtoken=addtoken, - GET_CSRF=GET_CSRF, - files=files, - do_error=do_error, - max_retry=max_retry + params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry ) def get_revids(self): @@ -347,7 +343,9 @@ def subcatquery_(self, **kwargs): new_tab2 = [] # --- if self.limit > 0 and len(self.result_table) >= self.limit: - printe.output(f"<> limit:{self.limit} reached, len of results: {len(self.result_table)} break ..") + printe.output( + f"<> limit:{self.limit} reached, len of results: {len(self.result_table)} break .." + ) break # --- depth_done += 1 diff --git a/newapi/super/S_Category/catdepth_new.py b/newapi/super/S_Category/catdepth_new.py index 2679d07..82ad3b0 100644 --- a/newapi/super/S_Category/catdepth_new.py +++ b/newapi/super/S_Category/catdepth_new.py @@ -1,8 +1,7 @@ -""" +""" """ -""" -import time import sys +import time from functools import lru_cache from ...api_utils import printe @@ -37,7 +36,7 @@ def args_group(title, kwargs): "with_lang": None, "tempyes": None, "props": None, - "only_titles": None + "only_titles": None, } # --- for k, v in kwargs.items(): @@ -58,7 +57,9 @@ def subcatquery(login_bot, title, sitecode=SITECODE, family=FAMILY, **kwargs): args2 = args_group(title, kwargs) # --- if print_s: - printe.output(f"<> catdepth_new.py sub cat query for {sitecode}:{title}, depth:{args2['depth']}, ns:{args2['ns']}, onlyns:{args2['onlyns']}") + printe.output( + f"<> catdepth_new.py sub cat query for {sitecode}:{title}, depth:{args2['depth']}, ns:{args2['ns']}, onlyns:{args2['onlyns']}" + ) # --- start = time.time() # --- @@ -77,6 +78,8 @@ def subcatquery(login_bot, title, sitecode=SITECODE, family=FAMILY, **kwargs): if print_s: lenpages = bot.get_len_pages() # --- - printe.output(f"<>catdepth_new.py: find {len(result)} pages({args2['ns']}) in {sitecode}:{title}, depth:{args2['depth']} in {delta} seconds | {lenpages=}") + printe.output( + f"<>catdepth_new.py: find {len(result)} pages({args2['ns']}) in {sitecode}:{title}, depth:{args2['depth']} in {delta} seconds | {lenpages=}" + ) # --- return result diff --git a/newapi/super/S_Page/bot.py b/newapi/super/S_Page/bot.py index 753004d..9607088 100644 --- a/newapi/super/S_Page/bot.py +++ b/newapi/super/S_Page/bot.py @@ -3,8 +3,10 @@ from .super.S_Page.bot import PAGE_APIS """ + from ..handel_errors import HANDEL_ERRORS + class PAGE_APIS(HANDEL_ERRORS): def __init__(self, login_bot): # print("class PAGE_APIS:") @@ -16,14 +18,9 @@ def __init__(self, login_bot): # --- super().__init__() - def post_continue(self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None): + def post_continue( + self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + ): return self.login_bot.post_continue( - params, - action, - _p_=_p_, - p_empty=p_empty, - Max=Max, - first=first, - _p_2=_p_2, - _p_2_empty=_p_2_empty + params, action, _p_=_p_, p_empty=p_empty, Max=Max, first=first, _p_2=_p_2, _p_2_empty=_p_2_empty ) diff --git a/newapi/super/S_Page/data.py b/newapi/super/S_Page/data.py index a6893df..cc95813 100644 --- a/newapi/super/S_Page/data.py +++ b/newapi/super/S_Page/data.py @@ -20,6 +20,8 @@ self\.(text_html|words|length|summary) self.content.$1 """ + + @dataclass class Content: # text: str = "" @@ -29,6 +31,7 @@ class Content: words: int = 0 length: int = 0 + r""" # meta self.is_Disambig = False @@ -45,6 +48,8 @@ class Content: self\.(is_Disambig|can_be_edit|userinfo|create_data|info|username|Exists|is_redirect|flagged|wikibase_item) self.meta.$1 """ + + @dataclass class Meta: is_Disambig: bool = False @@ -59,6 +64,7 @@ class Meta: flagged: str = "" wikibase_item: str = "" + r""" # revisions_data self.revid = "" @@ -72,6 +78,7 @@ class Meta: self.revisions_data.$1 """ + @dataclass class RevisionsData: revid: str = "" @@ -81,6 +88,7 @@ class RevisionsData: revisions: list = field(default_factory=list) touched: str = "" + r""" # LinksData self.back_links = [] @@ -93,6 +101,8 @@ class RevisionsData: self\.(back_links|links_here|extlinks|links|iwlinks) self.links_data.$1 """ + + @dataclass class LinksData: back_links: list = field(default_factory=list) @@ -112,12 +122,15 @@ class LinksData: self\.(categories|hidden_categories|all_categories_with_hidden) self.categories_data.$1 """ + + @dataclass class CategoriesData: categories: dict = field(default_factory=dict) hidden_categories: dict = field(default_factory=dict) all_categories_with_hidden: dict = field(default_factory=dict) + r""" # TemplateData self.templates = {} @@ -126,6 +139,8 @@ class CategoriesData: self\.(templates_API|templates) self.template_data.$1 """ + + @dataclass class TemplateData: templates: dict = field(default_factory=dict) diff --git a/newapi/super/S_Page/super_page.py b/newapi/super/S_Page/super_page.py index 0be33eb..9904caa 100644 --- a/newapi/super/S_Page/super_page.py +++ b/newapi/super/S_Page/super_page.py @@ -43,19 +43,20 @@ ''' """ + import os -from warnings import warn import sys -import wikitextparser as wtp +from warnings import warn -from .ar_err import find_edit_error -from .bot import PAGE_APIS -from .data import Content, Meta, RevisionsData, LinksData, CategoriesData, TemplateData +import wikitextparser as wtp -from ...api_utils import printe, txtlib, botEdit -from ...api_utils.except_err import exception_err, warn_err +from ...api_utils import botEdit, printe, txtlib from ...api_utils.ask_bot import ASK_BOT +from ...api_utils.except_err import exception_err, warn_err from ...api_utils.lang_codes import change_codes +from .ar_err import find_edit_error +from .bot import PAGE_APIS +from .data import CategoriesData, Content, LinksData, Meta, RevisionsData, TemplateData print_test = {1: "test" in sys.argv} @@ -98,13 +99,7 @@ def __init__(self, login_bot, title, lang="", family="wikipedia"): def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): # --- return self.login_bot.post_params( - params, - Type=Type, - addtoken=addtoken, - GET_CSRF=GET_CSRF, - files=files, - do_error=do_error, - max_retry=max_retry + params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry ) def false_edit(self): @@ -182,7 +177,7 @@ def find_create_data(self): "rvprop": "timestamp|ids|user", "rvslots": "*", "rvlimit": "1", - "rvdir": "newer" + "rvdir": "newer", } # --- data = self.post_params(params) @@ -195,9 +190,9 @@ def find_create_data(self): # --- if "parentid" in page_data and page_data["parentid"] == 0: self.meta.create_data = { - "timestamp" : page_data["timestamp"], - "user" : page_data.get("user", ""), - "anon" : page_data.get("anon", False), + "timestamp": page_data["timestamp"], + "user": page_data.get("user", ""), + "anon": page_data.get("anon", False), } # --- break @@ -267,9 +262,9 @@ def get_text(self, redirects=False): # --- if "parentid" in page_data and page_data["parentid"] == 0: self.meta.create_data = { - "timestamp" : page_data["timestamp"], - "user" : page_data.get("user", ""), - "anon" : page_data.get("anon", False), + "timestamp": page_data["timestamp"], + "user": page_data.get("user", ""), + "anon": page_data.get("anon", False), } # --- break @@ -599,7 +594,9 @@ def can_edit(self, script="", delay=0): if not self.text: self.text = self.get_text() # --- - self.meta.can_be_edit = botEdit.bot_May_Edit(text=self.text, title_page=self.title, botjob=script, page=self, delay=delay) + self.meta.can_be_edit = botEdit.bot_May_Edit( + text=self.text, title_page=self.title, botjob=script, page=self, delay=delay + ) # --- return self.meta.can_be_edit @@ -703,9 +700,20 @@ def save(self, newtext="", summary="", nocreate=1, minor="0", tags="", nodiff=Fa # --- message = f"Do you want to save this page? ({self.lang}:{self.title})" # --- - user = self.meta.username or getattr(self, 'user_login', '') - # --- - if self.ask_put(nodiff=nodiff, newtext=newtext, text=self.text, message=message, job="save", username=user, summary=self.content.summary) is False: + user = self.meta.username or getattr(self, "user_login", "") + # --- + if ( + self.ask_put( + nodiff=nodiff, + newtext=newtext, + text=self.text, + message=message, + job="save", + username=user, + summary=self.content.summary, + ) + is False + ): return False # --- params = { @@ -821,9 +829,12 @@ def Create(self, text="", summary="", nodiff="", noask=False): # --- message = f"Do you want to create this page? ({self.lang}:{self.title})" # --- - user = self.meta.username or getattr(self, 'user_login', '') + user = self.meta.username or getattr(self, "user_login", "") # --- - if self.ask_put(nodiff=nodiff, newtext=text, message=message, job="create", username=user, summary=summary) is False: + if ( + self.ask_put(nodiff=nodiff, newtext=text, message=message, job="create", username=user, summary=summary) + is False + ): return False # --- params = { diff --git a/newapi/super/__init__.py b/newapi/super/__init__.py index 2793e85..128ed0f 100644 --- a/newapi/super/__init__.py +++ b/newapi/super/__init__.py @@ -1,16 +1,14 @@ # -*- coding: utf-8 -*- -""" - -""" -from .S_API import bot_api +""" """ from . import super_login -from .S_Page import super_page +from .S_API import bot_api from .S_Category import catdepth_new +from .S_Page import super_page __all__ = [ - 'S_API', - 'bot_api', - 'super_page', - 'super_login', - 'catdepth_new', + "S_API", + "bot_api", + "super_page", + "super_login", + "catdepth_new", ] diff --git a/newapi/super/bot.py b/newapi/super/bot.py index 60daaa4..d4b094e 100644 --- a/newapi/super/bot.py +++ b/newapi/super/bot.py @@ -6,19 +6,19 @@ Exception:{'login': {'result': 'Failed', 'reason': 'You have made too many recent login attempts. Please wait 5 minutes before trying again.'}} """ -import sys -import os -import requests - -from .params_help import PARAMS_HELPS +import os +import sys from http.cookiejar import MozillaCookieJar +import requests + from ..api_utils import printe -from .cookies_bot import get_file_name, del_cookies_file from ..api_utils.except_err import exception_err -from .Login_db.bot import log_one from ..api_utils.user_agent import default_user_agent +from .cookies_bot import del_cookies_file, get_file_name +from .Login_db.bot import log_one +from .params_help import PARAMS_HELPS # cookies = get_cookies(lang, family, username) seasons_by_lang = {} @@ -71,7 +71,7 @@ def add_User_tables(self, family, table, lang="") -> None: if table["username"].find("bot") == -1 and family == "wikipedia": print(f"add_User_tables: {family=}, {table['username']=}") # --- - if family != "" and table['username'] != "" and table['password'] != "": + if family != "" and table["username"] != "" and table["password"] != "": # --- if self.family == family or (langx == "ar" and self.family.startswith("wik")): # wiktionary self.user_table_done = True @@ -111,7 +111,9 @@ def log_in(self) -> bool: Bot_passwords = self.password.find("@") != -1 logins_count[1] += 1 printe.output(f"<<{color}>> {botname}/page.py: Log_to_wiki {self.endpoint} count:{logins_count[1]}") - printe.output(f"{botname}/page.py: log to {self.lang}.{self.family}.org user:{self.username}, ({Bot_passwords=})") + printe.output( + f"{botname}/page.py: log to {self.lang}.{self.family}.org user:{self.username}, ({Bot_passwords=})" + ) logintoken = self.get_logintoken() @@ -142,7 +144,9 @@ def get_logintoken(self) -> str: self.log_error(r11.status_code, "logintoken") # --- if not str(r11.status_code).startswith("2"): - printe.output(f"<> {botname} {r11.status_code} Server Error: Server Hangup for url: {self.endpoint}") + printe.output( + f"<> {botname} {r11.status_code} Server Error: Server Hangup for url: {self.endpoint}" + ) # --- except Exception as e: exception_err(e) @@ -293,7 +297,9 @@ def _handle_server_error(self, req0, action, params=None): self.log_error(req0.status_code, action, params=params) # --- if not str(req0.status_code).startswith("2"): - printe.output(f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}") + printe.output( + f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}" + ) def raw_request(self, params, files=None, timeout=30): # --- @@ -411,7 +417,9 @@ def get_rest_result(self, url) -> dict: req0 = seasons_by_lang[self.sea_key].request("GET", url, headers=self.headers) # --- if not str(req0.status_code).startswith("2"): - printe.output(f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}") + printe.output( + f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}" + ) # --- except Exception as e: exception_err(e) diff --git a/newapi/super/bot_new.py b/newapi/super/bot_new.py index 5bb4eb2..006bbd5 100644 --- a/newapi/super/bot_new.py +++ b/newapi/super/bot_new.py @@ -5,23 +5,26 @@ Exception:{'login': {'result': 'Failed', 'reason': 'You have made too many recent login attempts. Please wait 5 minutes before trying again.'}} """ -import sys -import os + import copy -import requests +import os +import sys from http.cookiejar import MozillaCookieJar +import requests + from ..api_utils import printe from ..api_utils.except_err import exception_err -from .cookies_bot import get_file_name, del_cookies_file - -from .params_help import PARAMS_HELPS -from .Login_db.bot import log_one from ..api_utils.user_agent import default_user_agent -# import mwclient +from .cookies_bot import del_cookies_file, get_file_name +from .Login_db.bot import log_one # from mwclient.client import Site from .mwclient.client import Site +from .params_help import PARAMS_HELPS + +# import mwclient + # cookies = get_cookies(lang, family, username) logins_count = {1: 0} @@ -78,10 +81,14 @@ def __initialize_site(self): self.domain = f"{self.lang}.{self.family}.org" if "dopost" in sys.argv: - self.site_mwclient = Site(self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login) + self.site_mwclient = Site( + self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login + ) else: try: - self.site_mwclient = Site(self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login) + self.site_mwclient = Site( + self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login + ) except Exception as e: printe.output(f"Could not connect to ({self.domain}): %s" % e) return False @@ -161,6 +168,7 @@ def do_request(self, params=None, method="POST"): # ----- # ----- + class LOGIN_HELPS(MwClientSite, PARAMS_HELPS): def __init__(self) -> None: # --- @@ -188,7 +196,7 @@ def add_User_tables(self, family, table, lang="") -> None: if table["username"].find("bot") == -1 and family == "wikipedia": print(f"add_User_tables: {family=}, {table['username']=}") # --- - if family != "" and table['username'] != "" and table['password'] != "": + if family != "" and table["username"] != "" and table["password"] != "": # --- if self.family == family or (langx == "ar" and self.family.startswith("wik")): # wiktionary self.user_table_done = True diff --git a/newapi/super/cookies_bot.py b/newapi/super/cookies_bot.py index 5e6abfc..1467597 100644 --- a/newapi/super/cookies_bot.py +++ b/newapi/super/cookies_bot.py @@ -4,12 +4,14 @@ # cookies = get_cookies(lang, family, username) """ -import sys + import os import stat +import sys +from datetime import datetime, timedelta from functools import lru_cache from pathlib import Path -from datetime import datetime, timedelta + from ..api_utils import printe statgroup = stat.S_IRWXU | stat.S_IRWXG @@ -46,7 +48,7 @@ def get_file_name(lang, family, username) -> Path: # --- if "nocookies" in sys.argv: randome = os.urandom(8).hex() - return ta_dir /f"{randome}.txt" + return ta_dir / f"{randome}.txt" # --- lang = lang.lower() family = family.lower() diff --git a/newapi/super/handel_errors.py b/newapi/super/handel_errors.py index 1916be0..f060eb8 100644 --- a/newapi/super/handel_errors.py +++ b/newapi/super/handel_errors.py @@ -2,7 +2,9 @@ from .super.handel_errors import HANDEL_ERRORS """ + import sys + # from newapi import printe from ..api_utils import printe @@ -55,7 +57,12 @@ def handel_err(self, error: dict, function: str = "", params: dict = None, do_er abusefilter = error.get("abusefilter", "") description = abusefilter.get("description", "") printe.output(f"<> ** abusefilter-disallowed: {description} ") - if description in ["تأخير البوتات 3 ساعات", "تأخير البوتات 3 ساعات- 3 من 3", "تأخير البوتات 3 ساعات- 1 من 3", "تأخير البوتات 3 ساعات- 2 من 3"]: + if description in [ + "تأخير البوتات 3 ساعات", + "تأخير البوتات 3 ساعات- 3 من 3", + "تأخير البوتات 3 ساعات- 1 من 3", + "تأخير البوتات 3 ساعات- 2 من 3", + ]: return False return description # --- @@ -77,8 +84,8 @@ def handel_err(self, error: dict, function: str = "", params: dict = None, do_er return False # --- if do_error: - params['data'] = {} - params['text'] = {} + params["data"] = {} + params["text"] = {} printe.error(f"<>{function} ERROR: <>info: {err_info}, {params=}") # --- if "raise" in sys.argv: diff --git a/newapi/super/login_wrap.py b/newapi/super/login_wrap.py index 317b9e0..c93a891 100644 --- a/newapi/super/login_wrap.py +++ b/newapi/super/login_wrap.py @@ -8,11 +8,13 @@ # bots_login_cache.update(catbots_login2) """ + from ..api_utils import printe from .super_login import Login hases = {} + def LoginWrap(sitecode, family, bots_login_cache, User_tables): # --- cache_key = (sitecode, family) # Consider adding relevant kwargs to key @@ -30,7 +32,10 @@ def LoginWrap(sitecode, family, bots_login_cache, User_tables): hases[cache_key] += 1 # --- if hases[cache_key] % 100 == 0: - printe.output(f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {hases[cache_key]}", p=True) + printe.output( + f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {hases[cache_key]}", + p=True, + ) else: login_bot = Login(sitecode, family=family) # --- diff --git a/newapi/super/login_wrap_new.py b/newapi/super/login_wrap_new.py index 9134181..33fea41 100644 --- a/newapi/super/login_wrap_new.py +++ b/newapi/super/login_wrap_new.py @@ -7,7 +7,9 @@ # login_bot = LoginWrap(sitecode, family, User_tables) """ + from functools import lru_cache + from ..api_utils import printe from .super_login import Login @@ -45,7 +47,9 @@ def LoginWrap(sitecode, family, bots_login_cache, User_tables): # --- cache_info = _create_login_bot.cache_info() if cache_info.hits > 0 and cache_info.hits % 100 == 0: - printe.output(f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {cache_info.hits}", p=True) + printe.output( + f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {cache_info.hits}", p=True + ) # --- # Return bots_login_cache for backward compatibility return login_bot, bots_login_cache diff --git a/newapi/super/mwclient/__init__.py b/newapi/super/mwclient/__init__.py index f16d785..ee90946 100644 --- a/newapi/super/mwclient/__init__.py +++ b/newapi/super/mwclient/__init__.py @@ -1,34 +1,35 @@ """ - Copyright (c) 2006-2011 Bryan Tong Minh +Copyright (c) 2006-2011 Bryan Tong Minh - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. """ -from .errors import * # noqa: F401, F403 -from .client import Site, __version__ # noqa: F401 import logging import warnings +from .client import Site, __version__ # noqa: F401 +from .errors import * # noqa: F401, F403 + # Show DeprecationWarning -warnings.simplefilter('always', DeprecationWarning) +warnings.simplefilter("always", DeprecationWarning) logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/newapi/super/mwclient/client.py b/newapi/super/mwclient/client.py index c069443..b9935d9 100644 --- a/newapi/super/mwclient/client.py +++ b/newapi/super/mwclient/client.py @@ -1,25 +1,23 @@ -import warnings +import json import logging - +import warnings from collections import OrderedDict -import json import requests -from requests.auth import HTTPBasicAuth, AuthBase +from requests.auth import AuthBase, HTTPBasicAuth from requests_oauthlib import OAuth1 -from . import errors -from . import listing -# from .sleep import Sleepers -from .util import parse_timestamp, read_in_chunks, handle_limit from ..Login_db.bot import log_one +from . import errors, listing + +# from .sleep import Sleepers +from .util import handle_limit, parse_timestamp, read_in_chunks -__version__ = '0.11.0' +__version__ = "0.11.0" log = logging.getLogger(__name__) -USER_AGENT = 'mwclient/{} ({})'.format(__version__, - 'https://github.com/mwclient/mwclient') +USER_AGENT = "mwclient/{} ({})".format(__version__, "https://github.com/mwclient/mwclient") class Site: @@ -88,14 +86,34 @@ class Site: (where e is the exception object) and will be one of the API:Login errors. The most common error code is "Failed", indicating a wrong username or password. """ + api_limit = 500 - def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, - max_retries=25, wait_callback=lambda *x: None, clients_useragent=None, - max_lag=3, compress=True, force_login=True, do_init=True, httpauth=None, - connection_options=None, consumer_token=None, consumer_secret=None, - access_token=None, access_secret=None, client_certificate=None, - custom_headers=None, scheme='https', reqs=None): + def __init__( + self, + host, + path="/w/", + ext=".php", + pool=None, + retry_timeout=30, + max_retries=25, + wait_callback=lambda *x: None, + clients_useragent=None, + max_lag=3, + compress=True, + force_login=True, + do_init=True, + httpauth=None, + connection_options=None, + consumer_token=None, + consumer_secret=None, + access_token=None, + access_secret=None, + client_certificate=None, + custom_headers=None, + scheme="https", + reqs=None, + ): # Setup member variables self.host = host self.path = path @@ -106,19 +124,14 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, self.max_lag = str(max_lag) self.force_login = force_login if reqs and connection_options: - print(ValueError( - "reqs is a deprecated alias of connection_options. Do not specify both." - )) + print(ValueError("reqs is a deprecated alias of connection_options. Do not specify both.")) if reqs: - warnings.warn( - "reqs is deprecated in mwclient 1.0.0. Use connection_options instead", - DeprecationWarning - ) + warnings.warn("reqs is deprecated in mwclient 1.0.0. Use connection_options instead", DeprecationWarning) connection_options = reqs self.requests = connection_options or {} self.scheme = scheme - if 'timeout' not in self.requests: - self.requests['timeout'] = 30 # seconds + if "timeout" not in self.requests: + self.requests["timeout"] = 30 # seconds if consumer_token is not None: auth = OAuth1(consumer_token, consumer_secret, access_token, access_secret) @@ -126,24 +139,22 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, # workaround weird requests default to encode as latin-1 # https://github.com/mwclient/mwclient/issues/315 # https://github.com/psf/requests/issues/4564 - httpauth = [ - it.encode("utf-8") if isinstance(it, str) else it for it in httpauth - ] + httpauth = [it.encode("utf-8") if isinstance(it, str) else it for it in httpauth] auth = HTTPBasicAuth(*httpauth) elif httpauth is None or isinstance(httpauth, (AuthBase,)): auth = httpauth else: # FIXME: Raise a specific exception instead of a generic RuntimeError. - print(RuntimeError('Authentication is not a tuple or an instance of AuthBase')) + print(RuntimeError("Authentication is not a tuple or an instance of AuthBase")) # self.sleepers = Sleepers(max_retries, retry_timeout, wait_callback) # Site properties - self.blocked = False # Whether current user is blocked + self.blocked = False # Whether current user is blocked self.hasmsg = False # Whether current user has new messages - self.groups = [] # Groups current user belongs to - self.rights = [] # Rights current user has - self.tokens = {} # Edit tokens of the current user + self.groups = [] # Groups current user belongs to + self.rights = [] # Rights current user has + self.tokens = {} # Edit tokens of the current user self.version = None self.namespaces = self.default_namespaces @@ -157,10 +168,10 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, # Set User-Agent header field if clients_useragent: - ua = clients_useragent + ' ' + USER_AGENT + ua = clients_useragent + " " + USER_AGENT else: ua = USER_AGENT - self.connection.headers['User-Agent'] = ua + self.connection.headers["User-Agent"] = ua if custom_headers: self.connection.headers.update(custom_headers) @@ -187,11 +198,11 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, try: self.site_init() except errors.APIError as e: - if e.args[0] == 'mwoauth-invalid-authorization': + if e.args[0] == "mwoauth-invalid-authorization": print(errors.OAuthAuthorizationError(self, e.code, e.info)) # Private wiki, do init after login - if e.args[0] not in {'unknown_action', 'readapidenied'}: + if e.args[0] not in {"unknown_action", "readapidenied"}: # raise print("raise") @@ -201,39 +212,38 @@ def site_init(self): `do_init=False` constructor argument.""" if self.initialized: - info = self.get('query', meta='userinfo', uiprop='groups|rights') - userinfo = info['query']['userinfo'] - self.username = userinfo['name'] - self.groups = userinfo.get('groups', []) - self.rights = userinfo.get('rights', []) + info = self.get("query", meta="userinfo", uiprop="groups|rights") + userinfo = info["query"]["userinfo"] + self.username = userinfo["name"] + self.groups = userinfo.get("groups", []) + self.rights = userinfo.get("rights", []) self.tokens = {} return - meta = self.get('query', meta='siteinfo|userinfo', - siprop='general|namespaces', uiprop='groups|rights', - retry_on_error=False) + meta = self.get( + "query", meta="siteinfo|userinfo", siprop="general|namespaces", uiprop="groups|rights", retry_on_error=False + ) # Extract site info - self.site = meta['query']['general'] + self.site = meta["query"]["general"] self.namespaces = { - namespace['id']: namespace.get('*', '') - for namespace in meta['query']['namespaces'].values() + namespace["id"]: namespace.get("*", "") for namespace in meta["query"]["namespaces"].values() } - self.version = self.version_tuple_from_generator(self.site['generator']) + self.version = self.version_tuple_from_generator(self.site["generator"]) # Require MediaWiki version >= 1.16 self.require(1, 16) # User info - userinfo = meta['query']['userinfo'] - self.username = userinfo['name'] - self.groups = userinfo.get('groups', []) - self.rights = userinfo.get('rights', []) + userinfo = meta["query"]["userinfo"] + self.username = userinfo["name"] + self.groups = userinfo.get("groups", []) + self.rights = userinfo.get("rights", []) self.initialized = True @staticmethod - def version_tuple_from_generator(string, prefix='MediaWiki '): + def version_tuple_from_generator(string, prefix="MediaWiki "): """Return a version tuple from a MediaWiki Generator string. Example: @@ -248,9 +258,9 @@ def version_tuple_from_generator(string, prefix='MediaWiki '): A tuple containing the individual elements of the given version number. """ if not string.startswith(prefix): - print(errors.MediaWikiVersionError('Unknown generator {}'.format(string))) + print(errors.MediaWikiVersionError("Unknown generator {}".format(string))) - version = string[len(prefix):].split('.') + version = string[len(prefix) :].split(".") def split_num(s): """Split the string on the first non-digit character. @@ -261,28 +271,43 @@ def split_num(s): """ i = 0 while i < len(s): - if s[i] < '0' or s[i] > '9': + if s[i] < "0" or s[i] > "9": break i += 1 if s[i:]: - return (int(s[:i]), s[i:], ) + return ( + int(s[:i]), + s[i:], + ) else: - return (int(s[:i]), ) + return (int(s[:i]),) version_tuple = sum((split_num(s) for s in version), ()) if len(version_tuple) < 2: - print(errors.MediaWikiVersionError('Unknown MediaWiki {}' - .format('.'.join(version)))) + print(errors.MediaWikiVersionError("Unknown MediaWiki {}".format(".".join(version)))) return version_tuple default_namespaces = { - 0: '', 1: 'Talk', 2: 'User', 3: 'User talk', 4: 'Project', - 5: 'Project talk', 6: 'Image', 7: 'Image talk', 8: 'MediaWiki', - 9: 'MediaWiki talk', 10: 'Template', 11: 'Template talk', 12: 'Help', - 13: 'Help talk', 14: 'Category', 15: 'Category talk', - -1: 'Special', -2: 'Media' + 0: "", + 1: "Talk", + 2: "User", + 3: "User talk", + 4: "Project", + 5: "Project talk", + 6: "Image", + 7: "Image talk", + 8: "MediaWiki", + 9: "MediaWiki talk", + 10: "Template", + 11: "Template talk", + 12: "Help", + 13: "Help talk", + 14: "Category", + 15: "Category talk", + -1: "Special", + -2: "Media", } def __repr__(self): @@ -300,7 +325,7 @@ def get(self, action, *args, **kwargs): Returns: The raw response from the API call, as a dictionary. """ - return self.api(action, 'GET', *args, **kwargs) + return self.api(action, "GET", *args, **kwargs) def post(self, action, *args, **kwargs): """Perform a generic API call using POST. @@ -314,9 +339,9 @@ def post(self, action, *args, **kwargs): Returns: The raw response from the API call, as a dictionary. """ - return self.api(action, 'POST', *args, **kwargs) + return self.api(action, "POST", *args, **kwargs) - def api(self, action, http_method='POST', *args, **kwargs): + def api(self, action, http_method="POST", *args, **kwargs): """Perform a generic API call and handle errors. All arguments will be passed on. @@ -343,17 +368,17 @@ def api(self, action, http_method='POST', *args, **kwargs): """ kwargs.update(args) - if action == 'query' and 'continue' not in kwargs: - kwargs['continue'] = '' - if action == 'query': - if 'meta' in kwargs: - kwargs['meta'] += '|userinfo' + if action == "query" and "continue" not in kwargs: + kwargs["continue"] = "" + if action == "query": + if "meta" in kwargs: + kwargs["meta"] += "|userinfo" else: - kwargs['meta'] = 'userinfo' - if 'uiprop' in kwargs: - kwargs['uiprop'] += '|blockinfo|hasmsg' + kwargs["meta"] = "userinfo" + if "uiprop" in kwargs: + kwargs["uiprop"] += "|blockinfo|hasmsg" else: - kwargs['uiprop'] = 'blockinfo|hasmsg' + kwargs["uiprop"] = "blockinfo|hasmsg" # sleeper = self.sleepers.make() @@ -362,7 +387,7 @@ def api(self, action, http_method='POST', *args, **kwargs): if not info: info = {} # if self.handle_api_result(info, sleeper=sleeper): - self.handle_api_result(info)#, sleeper=sleeper + self.handle_api_result(info) # , sleeper=sleeper return info def log_error(self, result, action, params=None) -> None: @@ -386,59 +411,50 @@ def handle_api_result(self, info, kwargs=None, sleeper=None): # if sleeper is None: sleeper = self.sleepers.make() try: - userinfo = info['query']['userinfo'] + userinfo = info["query"]["userinfo"] except KeyError: userinfo = () - if 'blockedby' in userinfo: - self.blocked = (userinfo['blockedby'], userinfo.get('blockreason', '')) + if "blockedby" in userinfo: + self.blocked = (userinfo["blockedby"], userinfo.get("blockreason", "")) else: self.blocked = False - self.hasmsg = 'messages' in userinfo - self.logged_in = 'anon' not in userinfo and 'temp' not in userinfo - if 'warnings' in info: - for module, warning in info['warnings'].items(): - if '*' in warning: - log.warning(warning['*']) - - if 'error' in info: - if info['error'].get('code') in {'internal_api_error_DBConnectionError', - 'internal_api_error_DBQueryError'}: + self.hasmsg = "messages" in userinfo + self.logged_in = "anon" not in userinfo and "temp" not in userinfo + if "warnings" in info: + for module, warning in info["warnings"].items(): + if "*" in warning: + log.warning(warning["*"]) + + if "error" in info: + if info["error"].get("code") in {"internal_api_error_DBConnectionError", "internal_api_error_DBQueryError"}: # sleeper.sleep() return False # cope with https://phabricator.wikimedia.org/T106066 - if ( - info['error'].get('code') == 'mwoauth-invalid-authorization' - and 'Nonce already used' in info['error'].get('info') - ): - log.warning('Retrying due to nonce error, see' - 'https://phabricator.wikimedia.org/T106066') + if info["error"].get("code") == "mwoauth-invalid-authorization" and "Nonce already used" in info[ + "error" + ].get("info"): + log.warning("Retrying due to nonce error, see" "https://phabricator.wikimedia.org/T106066") # sleeper.sleep() return False - if 'query' in info['error']: + if "query" in info["error"]: # Semantic Mediawiki does not follow the standard error format - print(errors.APIError(None, info['error']['query'], kwargs)) + print(errors.APIError(None, info["error"]["query"], kwargs)) - if '*' in info['error']: - print(errors.APIError(info['error']['code'], - info['error']['info'], info['error']['*'])) - print(errors.APIError(info['error']['code'], - info['error']['info'], kwargs)) + if "*" in info["error"]: + print(errors.APIError(info["error"]["code"], info["error"]["info"], info["error"]["*"])) + print(errors.APIError(info["error"]["code"], info["error"]["info"], kwargs)) return True @staticmethod def _query_string(*args, **kwargs): kwargs.update(args) - qs1 = [ - (k, v) for k, v in kwargs.items() if k not in {'wpEditToken', 'token'} - ] - qs2 = [ - (k, v) for k, v in kwargs.items() if k in {'wpEditToken', 'token'} - ] + qs1 = [(k, v) for k, v in kwargs.items() if k not in {"wpEditToken", "token"}] + qs2 = [(k, v) for k, v in kwargs.items() if k in {"wpEditToken", "token"}] return OrderedDict(qs1 + qs2) - def raw_call(self, script, data, files=None, retry_on_error=True, http_method='POST'): + def raw_call(self, script, data, files=None, retry_on_error=True, http_method="POST"): """ Perform a generic request and return the raw text. @@ -471,40 +487,39 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method='P """ headers = {} if self.compress: - headers['Accept-Encoding'] = 'gzip' + headers["Accept-Encoding"] = "gzip" # sleeper = self.sleepers.make((script, data)) scheme = self.scheme host = self.host if isinstance(host, (list, tuple)): warnings.warn( - 'Specifying host as a tuple is deprecated as of mwclient 0.10.1. ' - + 'Please use the new scheme argument instead.', - DeprecationWarning + "Specifying host as a tuple is deprecated as of mwclient 0.10.1. " + + "Please use the new scheme argument instead.", + DeprecationWarning, ) scheme, host = host - url = '{scheme}://{host}{path}{script}{ext}'.format(scheme=scheme, host=host, - path=self.path, script=script, - ext=self.ext) + url = "{scheme}://{host}{path}{script}{ext}".format( + scheme=scheme, host=host, path=self.path, script=script, ext=self.ext + ) # while True: toraise = None wait_time = 0 - args = {'files': files, 'headers': headers} + args = {"files": files, "headers": headers} for k, v in self.requests.items(): args[k] = v - if http_method == 'GET': - args['params'] = data + if http_method == "GET": + args["params"] = data else: - args['data'] = data - maxlag = data.get('maxlag', self.max_lag) + args["data"] = data + maxlag = data.get("maxlag", self.max_lag) try: stream = self.connection.request(http_method, url, **args) - if stream.headers.get('x-database-lag'): - wait_time = int(stream.headers.get('retry-after')) - log.warning('Database lag exceeds max lag. ' - f'Waiting for {wait_time} seconds, maxlag:{maxlag}') + if stream.headers.get("x-database-lag"): + wait_time = int(stream.headers.get("retry-after")) + log.warning("Database lag exceeds max lag. " f"Waiting for {wait_time} seconds, maxlag:{maxlag}") # fall through to the sleep elif stream.status_code == 200: return stream.text @@ -513,25 +528,22 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method='P else: if not retry_on_error: stream.raise_for_status() - log.warning('Received {status} response: {text}. ' - 'Retrying in a moment.' - .format(status=stream.status_code, - text=stream.text)) + log.warning( + "Received {status} response: {text}. " + "Retrying in a moment.".format(status=stream.status_code, text=stream.text) + ) toraise = "stream" # fall through to the sleep return stream.text - except ( - requests.exceptions.ConnectionError, - requests.exceptions.Timeout - ) as err: + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as err: # In the event of a network problem # (e.g. DNS failure, refused connection, etc), # Requests will raise a ConnectionError exception. if not retry_on_error: print("raise") print(err) - log.warning('Connection error. Retrying in a moment.') + log.warning("Connection error. Retrying in a moment.") toraise = err # proceed to the sleep @@ -547,7 +559,7 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method='P else: print("raise") - def raw_api(self, action, http_method='POST', retry_on_error=True, *args, **kwargs): + def raw_api(self, action, http_method="POST", retry_on_error=True, *args, **kwargs): """Send a call to the API. Args: @@ -572,11 +584,10 @@ def raw_api(self, action, http_method='POST', retry_on_error=True, *args, **kwar performing the API request. requests.exceptions.Timeout: The API request timed out. """ - kwargs['action'] = action - kwargs['format'] = 'json' + kwargs["action"] = action + kwargs["format"] = "json" data = self._query_string(*args, **kwargs) - res = self.raw_call('api', data, retry_on_error=retry_on_error, - http_method=http_method) + res = self.raw_call("api", data, retry_on_error=retry_on_error, http_method=http_method) try: # data = json.loads(res, object_pairs_hook=OrderedDict) data2 = json.loads(res) @@ -588,11 +599,11 @@ def raw_api(self, action, http_method='POST', retry_on_error=True, *args, **kwar except ValueError: self.log_error("ValueError", action, params=data) # --- - if res.startswith('MediaWiki API is not enabled for this site.'): + if res.startswith("MediaWiki API is not enabled for this site."): print(errors.APIDisabledError) print(errors.InvalidResponse(res)) - def raw_index(self, action, http_method='POST', *args, **kwargs): + def raw_index(self, action, http_method="POST", *args, **kwargs): """Sends a call to index.php rather than the API. Args: @@ -614,11 +625,11 @@ def raw_index(self, action, http_method='POST', *args, **kwargs): performing the API request. requests.exceptions.Timeout: The API request timed out. """ - kwargs['action'] = action + kwargs["action"] = action data = self._query_string(*args, **kwargs) - if not data.get('maxlag'): - data['maxlag'] = self.max_lag - return self.raw_call('index', data, http_method=http_method) + if not data.get("maxlag"): + data["maxlag"] = self.max_lag + return self.raw_call("index", data, http_method=http_method) def require(self, major, minor, revision=None, raise_error=True): """Check whether the current wiki matches the required version. @@ -650,17 +661,19 @@ def require(self, major, minor, revision=None, raise_error=True): if raise_error is None: return # FIXME: Replace this with a specific error - print(RuntimeError('Site %s has not yet been initialized' % repr(self))) + print(RuntimeError("Site %s has not yet been initialized" % repr(self))) if revision is None: if self.version[:2] >= (major, minor): return True elif raise_error: - print(errors.MediaWikiVersionError( - 'Requires version {required[0]}.{required[1]}, ' - 'current version is {current[0]}.{current[1]}' - .format(required=(major, minor), - current=(self.version[:2]))) + print( + errors.MediaWikiVersionError( + "Requires version {required[0]}.{required[1]}, " + "current version is {current[0]}.{current[1]}".format( + required=(major, minor), current=(self.version[:2]) + ) + ) ) else: return False @@ -691,13 +704,12 @@ def email(self, user, text, subject, cc=False): EmailError (mwclient.errors.EmailError): Other email errors """ - token = self.get_token('email') + token = self.get_token("email") try: - info = self.post('emailuser', target=user, subject=subject, - text=text, ccme=cc, token=token) + info = self.post("emailuser", target=user, subject=subject, text=text, ccme=cc, token=token) except errors.APIError as e: - if e.args[0] == 'noemail': + if e.args[0] == "noemail": print(errors.NoSpecifiedEmail(user, e.args[1])) print(errors.EmailError(*e)) @@ -749,40 +761,35 @@ def login(self, username=None, password=None, cookies=None, domain=None): if self.credentials: # sleeper = self.sleepers.make() - kwargs = { - 'lgname': self.credentials[0], - 'lgpassword': self.credentials[1] - } + kwargs = {"lgname": self.credentials[0], "lgpassword": self.credentials[1]} if self.credentials[2]: - kwargs['lgdomain'] = self.credentials[2] + kwargs["lgdomain"] = self.credentials[2] # Try to login using the scheme for MW 1.27+. If the wiki is read protected, # it is not possible to get the wiki version upfront using the API, so we just # have to try. If the attempt fails, we try the old method. try: - kwargs['lgtoken'] = self.get_token('login') + kwargs["lgtoken"] = self.get_token("login") except (errors.APIError, KeyError): - log.debug('Failed to get login token, MediaWiki is older than 1.27.') - + log.debug("Failed to get login token, MediaWiki is older than 1.27.") # while True: - login = self.post('login', **kwargs) + login = self.post("login", **kwargs) - if login['login']['result'] == 'Success': - login_result = 'Success' + if login["login"]["result"] == "Success": + login_result = "Success" # break - elif login['login']['result'] == 'NeedToken': - login_result = 'NeedToken' - kwargs['lgtoken'] = login['login']['token'] - elif login['login']['result'] == 'Throttled': - login_result = 'Throttled' - so = int(login['login'].get('wait', 5)) + elif login["login"]["result"] == "NeedToken": + login_result = "NeedToken" + kwargs["lgtoken"] = login["login"]["token"] + elif login["login"]["result"] == "Throttled": + login_result = "Throttled" + so = int(login["login"].get("wait", 5)) # sleeper.sleep(so) print(f"so: {so}") else: - login_result = login['login']['result'] - print(errors.LoginError(self, login['login']['result'], - login['login']['reason'])) + login_result = login["login"]["result"] + print(errors.LoginError(self, login["login"]["result"], login["login"]["reason"])) self.site_init() @@ -836,26 +843,25 @@ def clientlogin(self, cookies=None, **kwargs): # Try to login using the scheme for MW 1.27+. If the wiki is read protected, # it is not possible to get the wiki version upfront using the API, so we just # have to try. If the attempt fails, we try the old method. - if 'logintoken' not in kwargs: + if "logintoken" not in kwargs: try: - kwargs['logintoken'] = self.get_token('login') + kwargs["logintoken"] = self.get_token("login") except (errors.APIError, KeyError): - log.debug('Failed to get login token, MediaWiki is older than 1.27.') + log.debug("Failed to get login token, MediaWiki is older than 1.27.") - if 'logincontinue' not in kwargs and 'loginreturnurl' not in kwargs: + if "logincontinue" not in kwargs and "loginreturnurl" not in kwargs: # should be great if API didn't require this... - kwargs['loginreturnurl'] = '%s://%s' % (self.scheme, self.host) + kwargs["loginreturnurl"] = "%s://%s" % (self.scheme, self.host) # while True: - login = self.post('clientlogin', **kwargs) - status = login['clientlogin'].get('status') - if status == 'PASS': + login = self.post("clientlogin", **kwargs) + status = login["clientlogin"].get("status") + if status == "PASS": return True - elif status in ('UI', 'REDIRECT'): - return login['clientlogin'] + elif status in ("UI", "REDIRECT"): + return login["clientlogin"] else: - print(errors.LoginError(self, status, - login['clientlogin'].get('message'))) + print(errors.LoginError(self, status, login["clientlogin"].get("message"))) def get_token(self, type, force=False, title=None): """Request a MediaWiki access token of the given `type`. @@ -876,41 +882,49 @@ def get_token(self, type, force=False, title=None): if self.version is None or self.version[:2] >= (1, 24): # The 'csrf' (cross-site request forgery) token introduced in 1.24 replaces # the majority of older tokens, like edittoken and movetoken. - if type not in {'watch', 'patrol', 'rollback', 'userrights', 'login'}: - type = 'csrf' + if type not in {"watch", "patrol", "rollback", "userrights", "login"}: + type = "csrf" if type not in self.tokens: - self.tokens[type] = '0' + self.tokens[type] = "0" - if self.tokens.get(type, '0') == '0' or force: + if self.tokens.get(type, "0") == "0" or force: if self.version is None or self.version[:2] >= (1, 24): # We use raw_api() rather than api() because api() is adding "userinfo" # to the query and this raises a readapideniederror if the wiki is read # protected, and we're trying to fetch a login token. - info = self.raw_api('query', 'GET', meta='tokens', type=type) + info = self.raw_api("query", "GET", meta="tokens", type=type) self.handle_api_result(info) # Note that for read protected wikis, we don't know the version when # fetching the login token. If it's < 1.27, the request below will # raise a KeyError that we should catch. - self.tokens[type] = info['query']['tokens']['%stoken' % type] + self.tokens[type] = info["query"]["tokens"]["%stoken" % type] else: if title is None: # Some dummy title was needed to get a token prior to 1.24 - title = 'Test' - info = self.post('query', titles=title, - prop='info', intoken=type) - for i in info['query']['pages'].values(): - if i['title'] == title: - self.tokens[type] = i['%stoken' % type] + title = "Test" + info = self.post("query", titles=title, prop="info", intoken=type) + for i in info["query"]["pages"].values(): + if i["title"] == title: + self.tokens[type] = i["%stoken" % type] return self.tokens[type] - def upload(self, file=None, filename=None, description='', ignore=False, - file_size=None, url=None, filekey=None, comment=None): + def upload( + self, + file=None, + filename=None, + description="", + ignore=False, + file_size=None, + url=None, + filekey=None, + comment=None, + ): """Upload a file to the site. Note that one of `file`, `filekey` and `url` must be specified, but not @@ -945,21 +959,16 @@ def upload(self, file=None, filename=None, description='', ignore=False, if file_size is not None: # Note that DeprecationWarning is hidden by default since Python 2.7 - warnings.warn( - 'file_size is deprecated since mwclient 0.7', - DeprecationWarning - ) + warnings.warn("file_size is deprecated since mwclient 0.7", DeprecationWarning) if filename is None: - print(TypeError('filename must be specified')) + print(TypeError("filename must be specified")) if len([x for x in [file, filekey, url] if x is not None]) != 1: - print(TypeError( - "exactly one of 'file', 'filekey' and 'url' must be specified" - )) + print(TypeError("exactly one of 'file', 'filekey' and 'url' must be specified")) image = self.Images[filename] - if not image.can('upload'): + if not image.can("upload"): print(errors.InsufficientPermission(filename)) if comment is None: @@ -970,8 +979,8 @@ def upload(self, file=None, filename=None, description='', ignore=False, text = description if file is not None: - if not hasattr(file, 'read'): - file = open(file, 'rb') + if not hasattr(file, "read"): + file = open(file, "rb") content_size = file.seek(0, 2) file.seek(0) @@ -980,25 +989,25 @@ def upload(self, file=None, filename=None, description='', ignore=False, return self.chunk_upload(file, filename, ignore, comment, text) predata = { - 'action': 'upload', - 'format': 'json', - 'filename': filename, - 'comment': comment, - 'text': text, - 'token': image.get_token('edit'), + "action": "upload", + "format": "json", + "filename": filename, + "comment": comment, + "text": text, + "token": image.get_token("edit"), } if ignore: - predata['ignorewarnings'] = 'true' + predata["ignorewarnings"] = "true" if url: - predata['url'] = url + predata["url"] = url # sessionkey was renamed to filekey in MediaWiki 1.18 # https://phabricator.wikimedia.org/rMW5f13517e36b45342f228f3de4298bb0fe186995d if self.version[:2] < (1, 18): - predata['sessionkey'] = filekey + predata["sessionkey"] = filekey else: - predata['filekey'] = filekey + predata["filekey"] = filekey postdata = predata files = None @@ -1009,25 +1018,27 @@ def upload(self, file=None, filename=None, description='', ignore=False, # Since the filename in Content-Disposition is not interpreted, # we can send some ascii-only dummy name rather than the real # filename, which might contain non-ascii. - files = {'file': ('fake-filename', file)} + files = {"file": ("fake-filename", file)} # sleeper = self.sleepers.make() # while True: - data = self.raw_call('api', postdata, files) + data = self.raw_call("api", postdata, files) info = json.loads(data) if not info: info = {} - if self.handle_api_result(info, kwargs=predata - #, sleeper=sleeper - ): - response = info.get('upload', {}) + if self.handle_api_result( + info, + kwargs=predata, + # , sleeper=sleeper + ): + response = info.get("upload", {}) # Workaround for https://github.com/mwclient/mwclient/issues/211 # ---------------------------------------------------------------- # Raise an error if the file already exists. This is necessary because # MediaWiki returns a warning, not an error, leading to silent failure. # The user must explicitly set ignore=True (ignorewarnings=True) to # overwrite an existing file. - if ignore is False and 'exists' in response.get('warnings', {}): + if ignore is False and "exists" in response.get("warnings", {}): print(errors.FileExists(filename)) # break @@ -1052,36 +1063,38 @@ def chunk_upload(self, file, filename, ignorewarnings, comment, text): file.seek(0) params = { - 'action': 'upload', - 'format': 'json', - 'stash': 1, - 'offset': 0, - 'filename': filename, - 'filesize': content_size, - 'token': image.get_token('edit'), + "action": "upload", + "format": "json", + "stash": 1, + "offset": 0, + "filename": filename, + "filesize": content_size, + "token": image.get_token("edit"), } if ignorewarnings: - params['ignorewarnings'] = 'true' + params["ignorewarnings"] = "true" # sleeper = self.sleepers.make() offset = 0 for chunk in read_in_chunks(file, self.chunk_size): while True: - data = self.raw_call('api', params, files={'chunk': chunk}) + data = self.raw_call("api", params, files={"chunk": chunk}) info = json.loads(data) - if self.handle_api_result(info, kwargs=params - #, sleeper=sleeper + if self.handle_api_result( + info, + kwargs=params, + # , sleeper=sleeper ): - response = info.get('upload', {}) + response = info.get("upload", {}) break offset += chunk.tell() chunk.close() - log.debug('%s: Uploaded %d of %d bytes', filename, offset, content_size) - params['filekey'] = response['filekey'] - if response['result'] == 'Continue': - params['offset'] = response['offset'] - elif response['result'] == 'Success': + log.debug("%s: Uploaded %d of %d bytes", filename, offset, content_size) + params["filekey"] = response["filekey"] + if response["result"] == "Continue": + params["offset"] = response["offset"] + elif response["result"] == "Success": file.close() break else: @@ -1091,15 +1104,14 @@ def chunk_upload(self, file, filename, ignorewarnings, comment, text): file.close() return response - del params['action'] - del params['stash'] - del params['offset'] - params['comment'] = comment - params['text'] = text - return self.post('upload', **params) + del params["action"] + del params["stash"] + del params["offset"] + params["comment"] = comment + params["text"] = text + return self.post("upload", **params) - def parse(self, text=None, title=None, page=None, prop=None, - redirects=False, mobileformat=False): + def parse(self, text=None, title=None, page=None, prop=None, redirects=False, mobileformat=False): """Parses the given content and returns parser output. Args: @@ -1119,19 +1131,19 @@ def parse(self, text=None, title=None, page=None, prop=None, """ kwargs = {} if text is not None: - kwargs['text'] = text + kwargs["text"] = text if title is not None: - kwargs['title'] = title + kwargs["title"] = title if page is not None: - kwargs['page'] = page + kwargs["page"] = page if prop is not None: - kwargs['prop'] = prop + kwargs["prop"] = prop if redirects: - kwargs['redirects'] = '1' + kwargs["redirects"] = "1" if mobileformat: - kwargs['mobileformat'] = '1' - result = self.post('parse', **kwargs) - return result['parse'] + kwargs["mobileformat"] = "1" + result = self.post("parse", **kwargs) + return result["parse"] # def block(self): TODO? # def unblock: TODO? @@ -1167,101 +1179,192 @@ def patrol(self, rcid=None, revid=None, tags=None): - ``tags`` requires at least MediaWiki 1.27. """ if self.require(1, 17, raise_error=False): - token = self.get_token('patrol') + token = self.get_token("patrol") else: # For MediaWiki versions earlier than 1.17, the patrol token is the same the # edit token. - token = self.get_token('edit') + token = self.get_token("edit") - result = self.post('patrol', rcid=rcid, revid=revid, tags=tags, token=token) - return result['patrol'] + result = self.post("patrol", rcid=rcid, revid=revid, tags=tags, token=token) + return result["patrol"] # Lists - def allpages(self, start=None, prefix=None, namespace='0', filterredir='all', - minsize=None, maxsize=None, prtype=None, prlevel=None, - limit=None, dir='ascending', filterlanglinks='all', generator=True, - end=None, max_items=None, api_chunk_size=None): + def allpages( + self, + start=None, + prefix=None, + namespace="0", + filterredir="all", + minsize=None, + maxsize=None, + prtype=None, + prlevel=None, + limit=None, + dir="ascending", + filterlanglinks="all", + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all pages on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('ap', generator) - kwargs = dict(listing.List.generate_kwargs( - pfx, ('from', start), ('to', end), prefix=prefix, - minsize=minsize, maxsize=maxsize, prtype=prtype, prlevel=prlevel, - namespace=namespace, filterredir=filterredir, dir=dir, - filterlanglinks=filterlanglinks, - )) - return listing.List.get_list(generator)(self, 'allpages', 'ap', - max_items=max_items, - api_chunk_size=api_chunk_size, - return_values='title', - **kwargs) - - def allimages(self, start=None, prefix=None, minsize=None, maxsize=None, limit=None, - dir='ascending', sha1=None, sha1base36=None, generator=True, end=None, - max_items=None, api_chunk_size=None): + pfx = listing.List.get_prefix("ap", generator) + kwargs = dict( + listing.List.generate_kwargs( + pfx, + ("from", start), + ("to", end), + prefix=prefix, + minsize=minsize, + maxsize=maxsize, + prtype=prtype, + prlevel=prlevel, + namespace=namespace, + filterredir=filterredir, + dir=dir, + filterlanglinks=filterlanglinks, + ) + ) + return listing.List.get_list(generator)( + self, "allpages", "ap", max_items=max_items, api_chunk_size=api_chunk_size, return_values="title", **kwargs + ) + + def allimages( + self, + start=None, + prefix=None, + minsize=None, + maxsize=None, + limit=None, + dir="ascending", + sha1=None, + sha1base36=None, + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all images on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('ai', generator) - kwargs = dict(listing.List.generate_kwargs( - pfx, ('from', start), ('to', end), prefix=prefix, - minsize=minsize, maxsize=maxsize, - dir=dir, sha1=sha1, sha1base36=sha1base36, - )) - return listing.List.get_list(generator)(self, 'allimages', 'ai', - max_items=max_items, - api_chunk_size=api_chunk_size, - return_values='timestamp|url', - **kwargs) - - def alllinks(self, start=None, prefix=None, unique=False, prop='title', - namespace='0', limit=None, generator=True, end=None, max_items=None, - api_chunk_size=None): + pfx = listing.List.get_prefix("ai", generator) + kwargs = dict( + listing.List.generate_kwargs( + pfx, + ("from", start), + ("to", end), + prefix=prefix, + minsize=minsize, + maxsize=maxsize, + dir=dir, + sha1=sha1, + sha1base36=sha1base36, + ) + ) + return listing.List.get_list(generator)( + self, + "allimages", + "ai", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="timestamp|url", + **kwargs, + ) + + def alllinks( + self, + start=None, + prefix=None, + unique=False, + prop="title", + namespace="0", + limit=None, + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve a list of all links on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('al', generator) - kwargs = dict(listing.List.generate_kwargs(pfx, ('from', start), ('to', end), - prefix=prefix, - prop=prop, namespace=namespace)) + pfx = listing.List.get_prefix("al", generator) + kwargs = dict( + listing.List.generate_kwargs( + pfx, ("from", start), ("to", end), prefix=prefix, prop=prop, namespace=namespace + ) + ) if unique: - kwargs[pfx + 'unique'] = '1' - return listing.List.get_list(generator)(self, 'alllinks', 'al', - max_items=max_items, - api_chunk_size=api_chunk_size, - return_values='title', **kwargs) - - def allcategories(self, start=None, prefix=None, dir='ascending', limit=None, - generator=True, end=None, max_items=None, api_chunk_size=None): + kwargs[pfx + "unique"] = "1" + return listing.List.get_list(generator)( + self, "alllinks", "al", max_items=max_items, api_chunk_size=api_chunk_size, return_values="title", **kwargs + ) + + def allcategories( + self, + start=None, + prefix=None, + dir="ascending", + limit=None, + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all categories on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('ac', generator) - kwargs = dict(listing.List.generate_kwargs(pfx, ('from', start), ('to', end), - prefix=prefix, dir=dir)) - return listing.List.get_list(generator)(self, 'allcategories', 'ac', - max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def allusers(self, start=None, prefix=None, group=None, prop=None, limit=None, - witheditsonly=False, activeusers=False, rights=None, end=None, - max_items=None, api_chunk_size=None): + pfx = listing.List.get_prefix("ac", generator) + kwargs = dict(listing.List.generate_kwargs(pfx, ("from", start), ("to", end), prefix=prefix, dir=dir)) + return listing.List.get_list(generator)( + self, "allcategories", "ac", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + ) + + def allusers( + self, + start=None, + prefix=None, + group=None, + prop=None, + limit=None, + witheditsonly=False, + activeusers=False, + rights=None, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all users on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('au', ('from', start), ('to', end), - prefix=prefix, - group=group, prop=prop, - rights=rights, - witheditsonly=witheditsonly, - activeusers=activeusers)) - return listing.List(self, 'allusers', 'au', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def blocks(self, start=None, end=None, dir='older', ids=None, users=None, limit=None, - prop='id|user|by|timestamp|expiry|reason|flags', max_items=None, - api_chunk_size=None): + kwargs = dict( + listing.List.generate_kwargs( + "au", + ("from", start), + ("to", end), + prefix=prefix, + group=group, + prop=prop, + rights=rights, + witheditsonly=witheditsonly, + activeusers=activeusers, + ) + ) + return listing.List(self, "allusers", "au", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def blocks( + self, + start=None, + end=None, + dir="older", + ids=None, + users=None, + limit=None, + prop="id|user|by|timestamp|expiry|reason|flags", + max_items=None, + api_chunk_size=None, + ): """Retrieve blocks as a generator. API doc: https://www.mediawiki.org/wiki/API:Blocks @@ -1288,23 +1391,30 @@ def blocks(self, start=None, end=None, dir='older', ids=None, users=None, limit= # TODO: Fix. Fix what? (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('bk', start=start, end=end, dir=dir, - ids=ids, users=users, prop=prop)) - return listing.List(self, 'blocks', 'bk', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def deletedrevisions(self, start=None, end=None, dir='older', namespace=None, - limit=None, prop='user|comment', max_items=None, - api_chunk_size=None): + kwargs = dict( + listing.List.generate_kwargs("bk", start=start, end=end, dir=dir, ids=ids, users=users, prop=prop) + ) + return listing.List(self, "blocks", "bk", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def deletedrevisions( + self, + start=None, + end=None, + dir="older", + namespace=None, + limit=None, + prop="user|comment", + max_items=None, + api_chunk_size=None, + ): # TODO: Fix (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('dr', start=start, end=end, dir=dir, - namespace=namespace, prop=prop)) - return listing.List(self, 'deletedrevs', 'dr', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) + kwargs = dict(listing.List.generate_kwargs("dr", start=start, end=end, dir=dir, namespace=namespace, prop=prop)) + return listing.List(self, "deletedrevs", "dr", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) - def exturlusage(self, query, prop=None, protocol='http', namespace=None, limit=None, - max_items=None, api_chunk_size=None): + def exturlusage( + self, query, prop=None, protocol="http", namespace=None, limit=None, max_items=None, api_chunk_size=None + ): r"""Retrieve the list of pages that link to a particular domain or URL, as a generator. @@ -1327,33 +1437,44 @@ def exturlusage(self, query, prop=None, protocol='http', namespace=None, limit=N """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('eu', query=query, prop=prop, - protocol=protocol, - namespace=namespace)) - return listing.List(self, 'exturlusage', 'eu', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def logevents(self, type=None, prop=None, start=None, end=None, - dir='older', user=None, title=None, limit=None, action=None, - max_items=None, api_chunk_size=None): + kwargs = dict( + listing.List.generate_kwargs("eu", query=query, prop=prop, protocol=protocol, namespace=namespace) + ) + return listing.List(self, "exturlusage", "eu", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def logevents( + self, + type=None, + prop=None, + start=None, + end=None, + dir="older", + user=None, + title=None, + limit=None, + action=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve logevents as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('le', prop=prop, type=type, - start=start, end=end, dir=dir, - user=user, title=title, action=action)) - return listing.List(self, 'logevents', 'le', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def checkuserlog(self, user=None, target=None, limit=None, dir='older', - start=None, end=None, max_items=None, api_chunk_size=10): + kwargs = dict( + listing.List.generate_kwargs( + "le", prop=prop, type=type, start=start, end=end, dir=dir, user=user, title=title, action=action + ) + ) + return listing.List(self, "logevents", "le", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def checkuserlog( + self, user=None, target=None, limit=None, dir="older", start=None, end=None, max_items=None, api_chunk_size=10 + ): """Retrieve checkuserlog items as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('cul', target=target, start=start, - end=end, dir=dir, user=user)) - return listing.NestedList('entries', self, 'checkuserlog', 'cul', - max_items=max_items, api_chunk_size=api_chunk_size, - **kwargs) + kwargs = dict(listing.List.generate_kwargs("cul", target=target, start=start, end=end, dir=dir, user=user)) + return listing.NestedList( + "entries", self, "checkuserlog", "cul", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + ) # def protectedtitles requires 1.15 def random(self, namespace, limit=None, max_items=None, api_chunk_size=20): @@ -1368,24 +1489,41 @@ def random(self, namespace, limit=None, max_items=None, api_chunk_size=20): """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('rn', namespace=namespace)) - return listing.List(self, 'random', 'rn', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def recentchanges(self, start=None, end=None, dir='older', namespace=None, - prop=None, show=None, limit=None, type=None, toponly=None, - max_items=None, api_chunk_size=None): - """List recent changes to the wiki, à la Special:Recentchanges. - """ + kwargs = dict(listing.List.generate_kwargs("rn", namespace=namespace)) + return listing.List(self, "random", "rn", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def recentchanges( + self, + start=None, + end=None, + dir="older", + namespace=None, + prop=None, + show=None, + limit=None, + type=None, + toponly=None, + max_items=None, + api_chunk_size=None, + ): + """List recent changes to the wiki, à la Special:Recentchanges.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('rc', start=start, end=end, dir=dir, - namespace=namespace, prop=prop, - show=show, type=type, - toponly='1' if toponly else None)) - return listing.List(self, 'recentchanges', 'rc', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def revisions(self, revids, prop='ids|timestamp|flags|comment|user'): + kwargs = dict( + listing.List.generate_kwargs( + "rc", + start=start, + end=end, + dir=dir, + namespace=namespace, + prop=prop, + show=show, + type=type, + toponly="1" if toponly else None, + ) + ) + return listing.List(self, "recentchanges", "rc", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def revisions(self, revids, prop="ids|timestamp|flags|comment|user"): """Get data about a list of revisions. See also the `Page.revisions()` method. @@ -1404,24 +1542,21 @@ def revisions(self, revids, prop='ids|timestamp|flags|comment|user'): Returns: A list of revisions """ - kwargs = { - 'prop': 'revisions', - 'rvprop': prop, - 'revids': '|'.join(map(str, revids)) - } + kwargs = {"prop": "revisions", "rvprop": prop, "revids": "|".join(map(str, revids))} revisions = [] - pages = self.get('query', **kwargs).get('query', {}).get('pages', {}).values() + pages = self.get("query", **kwargs).get("query", {}).get("pages", {}).values() for page in pages: - for revision in page.get('revisions', ()): - revision['pageid'] = page.get('pageid') - revision['pagetitle'] = page.get('title') - revision['timestamp'] = parse_timestamp(revision['timestamp']) + for revision in page.get("revisions", ()): + revision["pageid"] = page.get("pageid") + revision["pagetitle"] = page.get("title") + revision["timestamp"] = parse_timestamp(revision["timestamp"]) revisions.append(revision) return revisions - def search(self, search, namespace='0', what=None, redirects=False, limit=None, - max_items=None, api_chunk_size=None): + def search( + self, search, namespace="0", what=None, redirects=False, limit=None, max_items=None, api_chunk_size=None + ): """Perform a full text search. API doc: https://www.mediawiki.org/wiki/API:Search @@ -1447,39 +1582,62 @@ def search(self, search, namespace='0', what=None, redirects=False, limit=None, mwclient.listings.List: Search results iterator """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('sr', search=search, - namespace=namespace, what=what)) + kwargs = dict(listing.List.generate_kwargs("sr", search=search, namespace=namespace, what=what)) if redirects: - kwargs['srredirects'] = '1' - return listing.List(self, 'search', 'sr', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def usercontributions(self, user, start=None, end=None, dir='older', namespace=None, - prop=None, show=None, limit=None, uselang=None, max_items=None, - api_chunk_size=None): + kwargs["srredirects"] = "1" + return listing.List(self, "search", "sr", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def usercontributions( + self, + user, + start=None, + end=None, + dir="older", + namespace=None, + prop=None, + show=None, + limit=None, + uselang=None, + max_items=None, + api_chunk_size=None, + ): """ List the contributions made by a given user to the wiki. API doc: https://www.mediawiki.org/wiki/API:Usercontribs """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('uc', user=user, start=start, end=end, - dir=dir, namespace=namespace, - prop=prop, show=show)) - return listing.List(self, 'usercontribs', 'uc', max_items=max_items, - api_chunk_size=api_chunk_size, uselang=uselang, **kwargs) + kwargs = dict( + listing.List.generate_kwargs( + "uc", user=user, start=start, end=end, dir=dir, namespace=namespace, prop=prop, show=show + ) + ) + return listing.List( + self, "usercontribs", "uc", max_items=max_items, api_chunk_size=api_chunk_size, uselang=uselang, **kwargs + ) - def users(self, users, prop='blockinfo|groups|editcount'): + def users(self, users, prop="blockinfo|groups|editcount"): """ Get information about a list of users. API doc: https://www.mediawiki.org/wiki/API:Users """ - return listing.List(self, 'users', 'us', ususers='|'.join(users), usprop=prop) - - def watchlist(self, allrev=False, start=None, end=None, namespace=None, dir='older', - prop=None, show=None, limit=None, max_items=None, api_chunk_size=None): + return listing.List(self, "users", "us", ususers="|".join(users), usprop=prop) + + def watchlist( + self, + allrev=False, + start=None, + end=None, + namespace=None, + dir="older", + prop=None, + show=None, + limit=None, + max_items=None, + api_chunk_size=None, + ): """ List the pages on the current user's watchlist. @@ -1487,13 +1645,12 @@ def watchlist(self, allrev=False, start=None, end=None, namespace=None, dir='old """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('wl', start=start, end=end, - namespace=namespace, dir=dir, - prop=prop, show=show)) + kwargs = dict( + listing.List.generate_kwargs("wl", start=start, end=end, namespace=namespace, dir=dir, prop=prop, show=show) + ) if allrev: - kwargs['wlallrev'] = '1' - return listing.List(self, 'watchlist', 'wl', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) + kwargs["wlallrev"] = "1" + return listing.List(self, "watchlist", "wl", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) def expandtemplates(self, text, title=None, generatexml=False): """ @@ -1509,17 +1666,17 @@ def expandtemplates(self, text, title=None, generatexml=False): kwargs = {} if title is not None: - kwargs['title'] = title + kwargs["title"] = title if generatexml: # FIXME: Deprecated and replaced by `prop=parsetree`. - kwargs['generatexml'] = '1' + kwargs["generatexml"] = "1" - result = self.post('expandtemplates', text=text, **kwargs) + result = self.post("expandtemplates", text=text, **kwargs) if generatexml: - return result['expandtemplates']['*'], result['parsetree']['*'] + return result["expandtemplates"]["*"], result["parsetree"]["*"] else: - return result['expandtemplates']['*'] + return result["expandtemplates"]["*"] def ask(self, query, title=None): """ @@ -1545,15 +1702,16 @@ def ask(self, query, title=None): """ kwargs = {} if title is None: - kwargs['title'] = title + kwargs["title"] = title offset = 0 while offset is not None: - results = self.raw_api('ask', query='{query}|offset={offset}'.format( - query=query, offset=offset), http_method='GET', **kwargs) + results = self.raw_api( + "ask", query="{query}|offset={offset}".format(query=query, offset=offset), http_method="GET", **kwargs + ) self.handle_api_result(results) # raises APIError on error - offset = results.get('query-continue-offset') - answers = results['query'].get('results', []) + offset = results.get("query-continue-offset") + answers = results["query"].get("results", []) if isinstance(answers, dict): # In older versions of Semantic MediaWiki (at least until 2.3.0) diff --git a/newapi/super/mwclient/errors.py b/newapi/super/mwclient/errors.py index 5c87cca..963f015 100644 --- a/newapi/super/mwclient/errors.py +++ b/newapi/super/mwclient/errors.py @@ -58,17 +58,13 @@ def __init__(self, file_name): self.file_name = file_name def __str__(self): - return ('The file "{0}" already exists. Set ignore=True to overwrite it.' - .format(self.file_name)) + return 'The file "{0}" already exists. Set ignore=True to overwrite it.'.format(self.file_name) class LoginError(MwClientError): def __init__(self, site, code, info): - super(LoginError, self).__init__( - site, - {'result': code, 'reason': info} # For backwards-compability - ) + super(LoginError, self).__init__(site, {"result": code, "reason": info}) # For backwards-compability self.site = site self.code = code self.info = info @@ -84,11 +80,13 @@ class OAuthAuthorizationError(LoginError): class AssertUserFailedError(MwClientError): def __init__(self): - super(AssertUserFailedError, self).__init__(( - 'By default, mwclient protects you from accidentally editing ' - 'without being logged in. If you actually want to edit without ' - 'logging in, you can set force_login on the Site object to False.' - )) + super(AssertUserFailedError, self).__init__( + ( + "By default, mwclient protects you from accidentally editing " + "without being logged in. If you actually want to edit without " + "logging in, you can set force_login on the Site object to False." + ) + ) def __str__(self): return self.args[0] @@ -109,11 +107,13 @@ class NoWriteApi(MwClientError): class InvalidResponse(MwClientError): def __init__(self, response_text=None): - super(InvalidResponse, self).__init__(( - 'Did not get a valid JSON response from the server. Check that ' - 'you used the correct hostname. If you did, the server might ' - 'be wrongly configured or experiencing temporary problems.'), - response_text + super(InvalidResponse, self).__init__( + ( + "Did not get a valid JSON response from the server. Check that " + "you used the correct hostname. If you did, the server might " + "be wrongly configured or experiencing temporary problems." + ), + response_text, ) self.response_text = response_text diff --git a/newapi/super/mwclient/image.py b/newapi/super/mwclient/image.py index f3aef5b..6de17a8 100644 --- a/newapi/super/mwclient/image.py +++ b/newapi/super/mwclient/image.py @@ -1,21 +1,20 @@ +from . import listing, page from .util import handle_limit -from . import listing -from . import page class Image(page.Page): def __init__(self, site, name, info=None): super(Image, self).__init__( - site, name, info, extra_properties={ - 'imageinfo': ( - ('iiprop', - 'timestamp|user|comment|url|size|sha1|metadata|mime|archivename'), - ) - } + site, + name, + info, + extra_properties={ + "imageinfo": (("iiprop", "timestamp|user|comment|url|size|sha1|metadata|mime|archivename"),) + }, ) - self.imagerepository = self._info.get('imagerepository', '') - self.imageinfo = self._info.get('imageinfo', ({}, ))[0] + self.imagerepository = self._info.get("imagerepository", "") + self.imageinfo = self._info.get("imageinfo", ({},))[0] def imagehistory(self): """ @@ -24,32 +23,39 @@ def imagehistory(self): API doc: https://www.mediawiki.org/wiki/API:Imageinfo """ return listing.PageProperty( - self, 'imageinfo', 'ii', - iiprop='timestamp|user|comment|url|size|sha1|metadata|mime|archivename' + self, "imageinfo", "ii", iiprop="timestamp|user|comment|url|size|sha1|metadata|mime|archivename" ) - def imageusage(self, namespace=None, filterredir='all', redirect=False, - limit=None, generator=True, max_items=None, api_chunk_size=None): + def imageusage( + self, + namespace=None, + filterredir="all", + redirect=False, + limit=None, + generator=True, + max_items=None, + api_chunk_size=None, + ): """ List pages that use the given file. API doc: https://www.mediawiki.org/wiki/API:Imageusage """ - prefix = listing.List.get_prefix('iu', generator) - kwargs = dict(listing.List.generate_kwargs( - prefix, title=self.name, namespace=namespace, filterredir=filterredir - )) + prefix = listing.List.get_prefix("iu", generator) + kwargs = dict( + listing.List.generate_kwargs(prefix, title=self.name, namespace=namespace, filterredir=filterredir) + ) (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) if redirect: - kwargs['%sredirect' % prefix] = '1' + kwargs["%sredirect" % prefix] = "1" return listing.List.get_list(generator)( self.site, - 'imageusage', - 'iu', + "imageusage", + "iu", max_items=max_items, api_chunk_size=api_chunk_size, - return_values='title', - **kwargs + return_values="title", + **kwargs, ) def duplicatefiles(self, limit=None, max_items=None, api_chunk_size=None): @@ -62,13 +68,7 @@ def duplicatefiles(self, limit=None, max_items=None, api_chunk_size=None): not only specify the API chunk size. """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - return listing.PageProperty( - self, - 'duplicatefiles', - 'df', - max_items=max_items, - api_chunk_size=api_chunk_size - ) + return listing.PageProperty(self, "duplicatefiles", "df", max_items=max_items, api_chunk_size=api_chunk_size) def download(self, destination=None): """ @@ -84,7 +84,7 @@ def download(self, destination=None): Args: destination (file object): Destination file """ - url = self.imageinfo['url'] + url = self.imageinfo["url"] if destination is not None: res = self.site.connection.get(url, stream=True) for chunk in res.iter_content(1024): @@ -93,8 +93,4 @@ def download(self, destination=None): return self.site.connection.get(url).content def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.name, - self.site - ) + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) diff --git a/newapi/super/mwclient/listing.py b/newapi/super/mwclient/listing.py index 5626f28..d88a606 100644 --- a/newapi/super/mwclient/listing.py +++ b/newapi/super/mwclient/listing.py @@ -1,6 +1,5 @@ -from .util import parse_timestamp, handle_limit -from . import page -from . import image +from . import image, page +from .util import handle_limit, parse_timestamp class List: @@ -18,12 +17,21 @@ class List: to its misleading name. """ - def __init__(self, site, list_name, prefix, - limit=None, return_values=None, max_items=None, - api_chunk_size=None, *args, **kwargs): + def __init__( + self, + site, + list_name, + prefix, + limit=None, + return_values=None, + max_items=None, + api_chunk_size=None, + *args, + **kwargs, + ): self.site = site self.list_name = list_name - self.generator = 'list' + self.generator = "list" self.prefix = prefix kwargs.update(args) @@ -36,7 +44,7 @@ def __init__(self, site, list_name, prefix, # unneeded extra items (so long as it's below API limit) api_limit = site.api_limit api_chunk_size = api_chunk_size or min(max_items or api_limit, api_limit) - self.args[self.prefix + 'limit'] = str(api_chunk_size) + self.args[self.prefix + "limit"] = str(api_chunk_size) self.count = 0 self.max_items = max_items @@ -69,8 +77,8 @@ def __next__(self): self.load_chunk() self.count += 1 - if 'timestamp' in item: - item['timestamp'] = parse_timestamp(item['timestamp']) + if "timestamp" in item: + item["timestamp"] = parse_timestamp(item["timestamp"]) if isinstance(self, GeneratorList): return item @@ -96,45 +104,38 @@ def load_chunk(self): Else, set `self.last` to True. """ - data = self.site.get( - 'query', (self.generator, self.list_name), - *[(str(k), v) for k, v in self.args.items()] - ) + data = self.site.get("query", (self.generator, self.list_name), *[(str(k), v) for k, v in self.args.items()]) if not data: # Non existent page print(StopIteration) # Process response if not empty. # See: https://github.com/mwclient/mwclient/issues/194 - if 'query' in data: + if "query" in data: self.set_iter(data) - if data.get('continue'): + if data.get("continue"): # New style continuation, added in MediaWiki 1.21 - self.args.update(data['continue']) + self.args.update(data["continue"]) - elif self.list_name in data.get('query-continue', ()): + elif self.list_name in data.get("query-continue", ()): # Old style continuation - self.args.update(data['query-continue'][self.list_name]) + self.args.update(data["query-continue"][self.list_name]) else: self.last = True def set_iter(self, data): """Set `self._iter` to the API response `data`.""" - if self.result_member not in data['query']: + if self.result_member not in data["query"]: self._iter = iter(range(0)) - elif type(data['query'][self.result_member]) is list: - self._iter = iter(data['query'][self.result_member]) + elif type(data["query"][self.result_member]) is list: + self._iter = iter(data["query"][self.result_member]) else: - self._iter = iter(data['query'][self.result_member].values()) + self._iter = iter(data["query"][self.result_member].values()) def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.list_name, - self.site - ) + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.list_name, self.site) @staticmethod def generate_kwargs(_prefix, *args, **kwargs): @@ -145,7 +146,7 @@ def generate_kwargs(_prefix, *args, **kwargs): @staticmethod def get_prefix(prefix, generator=False): - return ('g' if generator else '') + prefix + return ("g" if generator else "") + prefix @staticmethod def get_list(generator=False): @@ -158,7 +159,7 @@ def __init__(self, nested_param, *args, **kwargs): self.nested_param = nested_param def set_iter(self, data): - self._iter = iter(data['query'][self.result_member][self.nested_param]) + self._iter = iter(data["query"][self.result_member][self.nested_param]) class GeneratorList(List): @@ -170,32 +171,31 @@ class GeneratorList(List): """ def __init__(self, site, list_name, prefix, *args, **kwargs): - super(GeneratorList, self).__init__(site, list_name, prefix, - *args, **kwargs) + super(GeneratorList, self).__init__(site, list_name, prefix, *args, **kwargs) - self.args['g' + self.prefix + 'limit'] = self.args[self.prefix + 'limit'] - del self.args[self.prefix + 'limit'] - self.generator = 'generator' + self.args["g" + self.prefix + "limit"] = self.args[self.prefix + "limit"] + del self.args[self.prefix + "limit"] + self.generator = "generator" - self.args['prop'] = 'info|imageinfo' - self.args['inprop'] = 'protection' + self.args["prop"] = "info|imageinfo" + self.args["inprop"] = "protection" - self.result_member = 'pages' + self.result_member = "pages" self.page_class = page.Page def __next__(self): info = super(GeneratorList, self).__next__() - if info['ns'] == 14: - return Category(self.site, '', info) - if info['ns'] == 6: - return image.Image(self.site, '', info) - return page.Page(self.site, '', info) + if info["ns"] == 14: + return Category(self.site, "", info) + if info["ns"] == 6: + return image.Image(self.site, "", info) + return page.Page(self.site, "", info) def load_chunk(self): # Put this here so that the constructor does not fail # on uninitialized sites - self.args['iiprop'] = 'timestamp|user|comment|url|size|sha1|metadata|archivename' + self.args["iiprop"] = "timestamp|user|comment|url|size|sha1|metadata|archivename" return super(GeneratorList, self).load_chunk() @@ -204,45 +204,42 @@ class Category(page.Page, GeneratorList): def __init__(self, site, name, info=None, namespace=None): page.Page.__init__(self, site, name, info) kwargs = {} - kwargs['gcmtitle'] = self.name + kwargs["gcmtitle"] = self.name if namespace: - kwargs['gcmnamespace'] = namespace - GeneratorList.__init__(self, site, 'categorymembers', 'cm', **kwargs) + kwargs["gcmnamespace"] = namespace + GeneratorList.__init__(self, site, "categorymembers", "cm", **kwargs) def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.name, - self.site + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) + + def members( + self, prop="ids|title", namespace=None, sort="sortkey", dir="asc", start=None, end=None, generator=True + ): + prefix = self.get_prefix("cm", generator) + kwargs = dict( + self.generate_kwargs( + prefix, prop=prop, namespace=namespace, sort=sort, dir=dir, start=start, end=end, title=self.name + ) ) - - def members(self, prop='ids|title', namespace=None, sort='sortkey', - dir='asc', start=None, end=None, generator=True): - prefix = self.get_prefix('cm', generator) - kwargs = dict(self.generate_kwargs(prefix, prop=prop, namespace=namespace, - sort=sort, dir=dir, start=start, end=end, - title=self.name)) - return self.get_list(generator)(self.site, 'categorymembers', 'cm', **kwargs) + return self.get_list(generator)(self.site, "categorymembers", "cm", **kwargs) class PageList(GeneratorList): - def __init__(self, site, prefix=None, start=None, namespace=0, redirects='all', - end=None): + def __init__(self, site, prefix=None, start=None, namespace=0, redirects="all", end=None): self.namespace = namespace kwargs = {} if prefix: - kwargs['gapprefix'] = prefix + kwargs["gapprefix"] = prefix if start: - kwargs['gapfrom'] = start + kwargs["gapfrom"] = start if end: - kwargs['gapto'] = end + kwargs["gapto"] = end - super(PageList, self).__init__(site, 'allpages', 'ap', - gapnamespace=str(namespace), - gapfilterredir=redirects, - **kwargs) + super(PageList, self).__init__( + site, "allpages", "ap", gapnamespace=str(namespace), gapfilterredir=redirects, **kwargs + ) def __getitem__(self, name): return self.get(name, None) @@ -258,7 +255,7 @@ def get(self, name, info=()): One of Category, Image or Page (default), according to namespace. """ if self.namespace != 0: - full_page_name = u"{namespace}:{name}".format( + full_page_name = "{namespace}:{name}".format( namespace=self.site.namespaces[self.namespace], name=name, ) @@ -293,11 +290,11 @@ def guess_namespace(self, name): for ns in self.site.namespaces: if ns == 0: continue - namespace = '%s:' % self.site.namespaces[ns].replace(' ', '_') + namespace = "%s:" % self.site.namespaces[ns].replace(" ", "_") if name.startswith(namespace): return ns elif ns in self.site.default_namespaces: - namespace = '%s:' % self.site.default_namespaces[ns].replace(' ', '_') + namespace = "%s:" % self.site.default_namespaces[ns].replace(" ", "_") if name.startswith(namespace): return ns return 0 @@ -306,15 +303,13 @@ def guess_namespace(self, name): class PageProperty(List): def __init__(self, page, prop, prefix, *args, **kwargs): - super(PageProperty, self).__init__(page.site, prop, prefix, - titles=page.name, - *args, **kwargs) + super(PageProperty, self).__init__(page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page - self.generator = 'prop' + self.generator = "prop" def set_iter(self, data): - for page in data['query']['pages'].values(): - if page['title'] == self.page.name: + for page in data["query"]["pages"].values(): + if page["title"] == self.page.name: self._iter = iter(page.get(self.list_name, ())) return print(StopIteration) @@ -323,15 +318,13 @@ def set_iter(self, data): class PagePropertyGenerator(GeneratorList): def __init__(self, page, prop, prefix, *args, **kwargs): - super(PagePropertyGenerator, self).__init__(page.site, prop, prefix, - titles=page.name, - *args, **kwargs) + super(PagePropertyGenerator, self).__init__(page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page class RevisionsIterator(PageProperty): def load_chunk(self): - if 'rvstartid' in self.args and 'rvstart' in self.args: - del self.args['rvstart'] + if "rvstartid" in self.args and "rvstart" in self.args: + del self.args["rvstart"] return super(RevisionsIterator, self).load_chunk() diff --git a/newapi/super/mwclient/page.py b/newapi/super/mwclient/page.py index 04080e7..fa66b0d 100644 --- a/newapi/super/mwclient/page.py +++ b/newapi/super/mwclient/page.py @@ -1,8 +1,8 @@ import time -from .util import parse_timestamp, handle_limit -from . import listing -from . import errors +from . import errors, listing +from .util import handle_limit, parse_timestamp + class Page: @@ -16,67 +16,61 @@ def __init__(self, site, name, info=None, extra_properties=None): if not info: if extra_properties: - prop = 'info|' + '|'.join(extra_properties.keys()) + prop = "info|" + "|".join(extra_properties.keys()) extra_props = [] for extra_prop in extra_properties.values(): extra_props.extend(extra_prop) else: - prop = 'info' + prop = "info" extra_props = () if type(name) is int: - info = self.site.get('query', prop=prop, pageids=name, - inprop='protection', *extra_props) + info = self.site.get("query", prop=prop, pageids=name, inprop="protection", *extra_props) else: - info = self.site.get('query', prop=prop, titles=name, - inprop='protection', *extra_props) - info = next(iter(info['query']['pages'].values())) + info = self.site.get("query", prop=prop, titles=name, inprop="protection", *extra_props) + info = next(iter(info["query"]["pages"].values())) self._info = info - if 'invalid' in info: - print(errors.InvalidPageTitle(info.get('invalidreason'))) + if "invalid" in info: + print(errors.InvalidPageTitle(info.get("invalidreason"))) - self.namespace = info.get('ns', 0) - self.name = info.get('title', '') + self.namespace = info.get("ns", 0) + self.name = info.get("title", "") if self.namespace: self.page_title = self.strip_namespace(self.name) else: self.page_title = self.name - self.base_title = self.page_title.split('/')[0] - self.base_name = self.name.split('/')[0] - - self.touched = parse_timestamp(info.get('touched')) - self.revision = info.get('lastrevid', 0) - self.exists = 'missing' not in info - self.length = info.get('length') - self.protection = { - i['type']: (i['level'], i.get('expiry')) - for i in info.get('protection', ()) - if i - } - self.redirect = 'redirect' in info - self.pageid = info.get('pageid', None) - self.contentmodel = info.get('contentmodel', None) - self.pagelanguage = info.get('pagelanguage', None) - self.restrictiontypes = info.get('restrictiontypes', None) + self.base_title = self.page_title.split("/")[0] + self.base_name = self.name.split("/")[0] + + self.touched = parse_timestamp(info.get("touched")) + self.revision = info.get("lastrevid", 0) + self.exists = "missing" not in info + self.length = info.get("length") + self.protection = {i["type"]: (i["level"], i.get("expiry")) for i in info.get("protection", ()) if i} + self.redirect = "redirect" in info + self.pageid = info.get("pageid", None) + self.contentmodel = info.get("contentmodel", None) + self.pagelanguage = info.get("pagelanguage", None) + self.restrictiontypes = info.get("restrictiontypes", None) self.last_rev_time = None self.edit_time = None def redirects_to(self): - """ Get the redirect target page, or None if the page is not a redirect.""" - info = self.site.get('query', prop='pageprops', titles=self.name, redirects='') - if 'redirects' in info['query']: - for page in info['query']['redirects']: - if page['from'] == self.name: - return Page(self.site, page['to']) + """Get the redirect target page, or None if the page is not a redirect.""" + info = self.site.get("query", prop="pageprops", titles=self.name, redirects="") + if "redirects" in info["query"]: + for page in info["query"]["redirects"]: + if page["from"] == self.name: + return Page(self.site, page["to"]) return None else: return None def resolve_redirect(self): - """ Get the redirect target page, or the current page if its not a redirect.""" + """Get the redirect target page, or the current page if its not a redirect.""" target_page = self.redirects_to() if target_page is None: return self @@ -84,26 +78,22 @@ def resolve_redirect(self): return target_page def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.name, - self.site - ) + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) @staticmethod def strip_namespace(title): - if title[0] == ':': + if title[0] == ":": title = title[1:] - return title[title.find(':') + 1:] + return title[title.find(":") + 1 :] @staticmethod def normalize_title(title): # TODO: Make site dependent title = title.strip() - if title[0] == ':': + if title[0] == ":": title = title[1:] title = title[0].upper() + title[1:] - title = title.replace(' ', '_') + title = title.replace(" ", "_") return title def can(self, action): @@ -116,15 +106,15 @@ def can(self, action): """ level = self.protection.get(action, (action,))[0] - if level == 'sysop': - level = 'editprotected' + if level == "sysop": + level = "editprotected" return level in self.site.rights def get_token(self, type, force=False): return self.site.get_token(type, force, title=self.name) - def text(self, section=None, expandtemplates=False, cache=True, slot='main'): + def text(self, section=None, expandtemplates=False, cache=True, slot="main"): """Get the current wikitext of the page, or of a specific section. If the page does not exist, an empty string is returned. By @@ -139,10 +129,10 @@ def text(self, section=None, expandtemplates=False, cache=True, slot='main'): cache (bool): Use in-memory caching (default: `True`) """ - if not self.can('read'): + if not self.can("read"): print(errors.InsufficientPermission(self)) if not self.exists: - return '' + return "" if section is not None: section = str(section) @@ -153,17 +143,16 @@ def text(self, section=None, expandtemplates=False, cache=True, slot='main'): # we set api_chunk_size not max_items because otherwise revisions' # default api_chunk_size of 50 gets used and we get 50 revisions; # no need to set max_items as well as we only iterate one time - revs = self.revisions(prop='content|timestamp', api_chunk_size=1, section=section, - slots=slot) + revs = self.revisions(prop="content|timestamp", api_chunk_size=1, section=section, slots=slot) try: rev = next(revs) - if 'slots' in rev: - text = rev['slots'][slot]['*'] + if "slots" in rev: + text = rev["slots"][slot]["*"] else: - text = rev['*'] - self.last_rev_time = rev['timestamp'] + text = rev["*"] + self.last_rev_time = rev["timestamp"] except StopIteration: - text = '' + text = "" self.last_rev_time = None if not expandtemplates: self.edit_time = time.gmtime() @@ -180,21 +169,16 @@ def save(self, *args, **kwargs): """Alias for edit, for maintaining backwards compatibility.""" return self.edit(*args, **kwargs) - def edit(self, text, summary='', minor=False, bot=True, section=None, **kwargs): - """Update the text of a section or the whole page by performing an edit operation. - """ + def edit(self, text, summary="", minor=False, bot=True, section=None, **kwargs): + """Update the text of a section or the whole page by performing an edit operation.""" return self._edit(summary, minor, bot, section, text=text, **kwargs) - def append(self, text, summary='', minor=False, bot=True, section=None, - **kwargs): - """Append text to a section or the whole page by performing an edit operation. - """ + def append(self, text, summary="", minor=False, bot=True, section=None, **kwargs): + """Append text to a section or the whole page by performing an edit operation.""" return self._edit(summary, minor, bot, section, appendtext=text, **kwargs) - def prepend(self, text, summary='', minor=False, bot=True, section=None, - **kwargs): - """Prepend text to a section or the whole page by performing an edit operation. - """ + def prepend(self, text, summary="", minor=False, bot=True, section=None, **kwargs): + """Prepend text to a section or the whole page by performing an edit operation.""" return self._edit(summary, minor, bot, section, prependtext=text, **kwargs) def _edit(self, summary, minor, bot, section, **kwargs): @@ -202,42 +186,40 @@ def _edit(self, summary, minor, bot, section, **kwargs): print(errors.AssertUserFailedError()) if self.site.blocked: print(errors.UserBlocked(self.site.blocked)) - if not self.can('edit'): + if not self.can("edit"): print(errors.ProtectedPageError(self)) data = {} if minor: - data['minor'] = '1' + data["minor"] = "1" if not minor: - data['notminor'] = '1' + data["notminor"] = "1" if self.last_rev_time: - data['basetimestamp'] = time.strftime('%Y%m%d%H%M%S', self.last_rev_time) + data["basetimestamp"] = time.strftime("%Y%m%d%H%M%S", self.last_rev_time) if self.edit_time: - data['starttimestamp'] = time.strftime('%Y%m%d%H%M%S', self.edit_time) + data["starttimestamp"] = time.strftime("%Y%m%d%H%M%S", self.edit_time) if bot: - data['bot'] = '1' + data["bot"] = "1" if section is not None: - data['section'] = section + data["section"] = section data.update(kwargs) if self.site.force_login: - data['assert'] = 'user' + data["assert"] = "user" def do_edit(): - result = self.site.post('edit', title=self.name, summary=summary, - token=self.get_token('edit'), - **data) - if result['edit'].get('result').lower() == 'failure': - print(errors.EditError(self, result['edit'])) + result = self.site.post("edit", title=self.name, summary=summary, token=self.get_token("edit"), **data) + if result["edit"].get("result").lower() == "failure": + print(errors.EditError(self, result["edit"])) return result try: result = do_edit() except errors.APIError as e: - if e.code == 'badtoken': + if e.code == "badtoken": # Retry, but only once to avoid an infinite loop - self.get_token('edit', force=True) + self.get_token("edit", force=True) try: result = do_edit() except errors.APIError as e: @@ -246,29 +228,37 @@ def do_edit(): self.handle_edit_error(e, summary) # 'newtimestamp' is not included if no change was made - if 'newtimestamp' in result['edit'].keys(): - self.last_rev_time = parse_timestamp(result['edit'].get('newtimestamp')) + if "newtimestamp" in result["edit"].keys(): + self.last_rev_time = parse_timestamp(result["edit"].get("newtimestamp")) # Workaround for https://phabricator.wikimedia.org/T211233 for cookie in self.site.connection.cookies: - if 'PostEditRevision' in cookie.name: - self.site.connection.cookies.clear(cookie.domain, cookie.path, - cookie.name) + if "PostEditRevision" in cookie.name: + self.site.connection.cookies.clear(cookie.domain, cookie.path, cookie.name) # clear the page text cache self._textcache = {} - return result['edit'] + return result["edit"] def handle_edit_error(self, e, summary): - if e.code == 'editconflict': + if e.code == "editconflict": print(errors.EditError(self, summary, e.info)) - elif e.code in {'protectedtitle', 'cantcreate', 'cantcreate-anon', - 'noimageredirect-anon', 'noimageredirect', 'noedit-anon', - 'noedit', 'protectedpage', 'cascadeprotected', - 'customcssjsprotected', - 'protectednamespace-interface', 'protectednamespace'}: + elif e.code in { + "protectedtitle", + "cantcreate", + "cantcreate-anon", + "noimageredirect-anon", + "noimageredirect", + "noedit-anon", + "noedit", + "protectedpage", + "cascadeprotected", + "customcssjsprotected", + "protectednamespace-interface", + "protectednamespace", + }: print(errors.ProtectedPageError(self, e.code, e.info)) - elif e.code == 'assertuserfailed': + elif e.code == "assertuserfailed": print(errors.AssertUserFailedError()) else: print(e) @@ -281,10 +271,9 @@ def touch(self): """ if not self.exists: return - self.append('') + self.append("") - def move(self, new_title, reason='', move_talk=True, no_redirect=False, - move_subpages=False, ignore_warnings=False): + def move(self, new_title, reason="", move_talk=True, no_redirect=False, move_subpages=False, ignore_warnings=False): """Move (rename) page to new_title. If user account is an administrator, specify no_redirect as True to not @@ -294,73 +283,89 @@ def move(self, new_title, reason='', move_talk=True, no_redirect=False, exception is raised. """ - if not self.can('move'): + if not self.can("move"): print(errors.InsufficientPermission(self)) data = {} if move_talk: - data['movetalk'] = '1' + data["movetalk"] = "1" if no_redirect: - data['noredirect'] = '1' + data["noredirect"] = "1" if move_subpages: - data['movesubpages'] = '1' + data["movesubpages"] = "1" if ignore_warnings: - data['ignorewarnings'] = '1' - result = self.site.post('move', ('from', self.name), to=new_title, - token=self.get_token('move'), reason=reason, **data) - return result['move'] + data["ignorewarnings"] = "1" + result = self.site.post( + "move", ("from", self.name), to=new_title, token=self.get_token("move"), reason=reason, **data + ) + return result["move"] - def delete(self, reason='', watch=False, unwatch=False, oldimage=False): + def delete(self, reason="", watch=False, unwatch=False, oldimage=False): """Delete page. If user does not have permission to delete page, an InsufficientPermission exception is raised. """ - if not self.can('delete'): + if not self.can("delete"): print(errors.InsufficientPermission(self)) data = {} if watch: - data['watch'] = '1' + data["watch"] = "1" if unwatch: - data['unwatch'] = '1' + data["unwatch"] = "1" if oldimage: - data['oldimage'] = oldimage - result = self.site.post('delete', title=self.name, - token=self.get_token('delete'), - reason=reason, **data) - return result['delete'] + data["oldimage"] = oldimage + result = self.site.post("delete", title=self.name, token=self.get_token("delete"), reason=reason, **data) + return result["delete"] def purge(self): """Purge server-side cache of page. This will re-render templates and other dynamic content. """ - self.site.post('purge', titles=self.name) + self.site.post("purge", titles=self.name) # def watch: requires 1.14 # Properties - def backlinks(self, namespace=None, filterredir='all', redirect=False, - limit=None, generator=True, max_items=None, api_chunk_size=None): + def backlinks( + self, + namespace=None, + filterredir="all", + redirect=False, + limit=None, + generator=True, + max_items=None, + api_chunk_size=None, + ): """List pages that link to the current page, similar to Special:Whatlinkshere. API doc: https://www.mediawiki.org/wiki/API:Backlinks """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - prefix = listing.List.get_prefix('bl', generator) - kwargs = dict(listing.List.generate_kwargs( - prefix, namespace=namespace, filterredir=filterredir, - )) + prefix = listing.List.get_prefix("bl", generator) + kwargs = dict( + listing.List.generate_kwargs( + prefix, + namespace=namespace, + filterredir=filterredir, + ) + ) if redirect: - kwargs['%sredirect' % prefix] = '1' - kwargs[prefix + 'title'] = self.name + kwargs["%sredirect" % prefix] = "1" + kwargs[prefix + "title"] = self.name return listing.List.get_list(generator)( - self.site, 'backlinks', 'bl', max_items=max_items, - api_chunk_size=api_chunk_size, return_values='title', **kwargs + self.site, + "backlinks", + "bl", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="title", + **kwargs, ) def categories(self, generator=True, show=None): @@ -376,23 +381,18 @@ def categories(self, generator=True, show=None): Returns: listings.PagePropertyGenerator """ - prefix = listing.List.get_prefix('cl', generator) - kwargs = dict(listing.List.generate_kwargs( - prefix, show=show - )) + prefix = listing.List.get_prefix("cl", generator) + kwargs = dict(listing.List.generate_kwargs(prefix, show=show)) if generator: - return listing.PagePropertyGenerator( - self, 'categories', 'cl', **kwargs - ) + return listing.PagePropertyGenerator(self, "categories", "cl", **kwargs) else: # TODO: return sortkey if wanted - return listing.PageProperty( - self, 'categories', 'cl', return_values='title', **kwargs - ) + return listing.PageProperty(self, "categories", "cl", return_values="title", **kwargs) - def embeddedin(self, namespace=None, filterredir='all', limit=None, generator=True, - max_items=None, api_chunk_size=None): + def embeddedin( + self, namespace=None, filterredir="all", limit=None, generator=True, max_items=None, api_chunk_size=None + ): """List pages that transclude the current page. API doc: https://www.mediawiki.org/wiki/API:Embeddedin @@ -410,14 +410,18 @@ def embeddedin(self, namespace=None, filterredir='all', limit=None, generator=Tr listings.List: Page iterator """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - prefix = listing.List.get_prefix('ei', generator) - kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace, - filterredir=filterredir)) - kwargs[prefix + 'title'] = self.name + prefix = listing.List.get_prefix("ei", generator) + kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace, filterredir=filterredir)) + kwargs[prefix + "title"] = self.name return listing.List.get_list(generator)( - self.site, 'embeddedin', 'ei', max_items=max_items, - api_chunk_size=api_chunk_size, return_values='title', **kwargs + self.site, + "embeddedin", + "ei", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="title", + **kwargs, ) def extlinks(self): @@ -426,7 +430,7 @@ def extlinks(self): API doc: https://www.mediawiki.org/wiki/API:Extlinks """ - return listing.PageProperty(self, 'extlinks', 'el', return_values='*') + return listing.PageProperty(self, "extlinks", "el", return_values="*") def images(self, generator=True): """List files/images embedded in the current page. @@ -435,10 +439,9 @@ def images(self, generator=True): """ if generator: - return listing.PagePropertyGenerator(self, 'images', '') + return listing.PagePropertyGenerator(self, "images", "") else: - return listing.PageProperty(self, 'images', '', - return_values='title') + return listing.PageProperty(self, "images", "", return_values="title") def iwlinks(self): """List interwiki links from the current page. @@ -446,8 +449,7 @@ def iwlinks(self): API doc: https://www.mediawiki.org/wiki/API:Iwlinks """ - return listing.PageProperty(self, 'iwlinks', 'iw', - return_values=('prefix', '*')) + return listing.PageProperty(self, "iwlinks", "iw", return_values=("prefix", "*")) def langlinks(self, **kwargs): """List interlanguage links from the current page. @@ -455,9 +457,7 @@ def langlinks(self, **kwargs): API doc: https://www.mediawiki.org/wiki/API:Langlinks """ - return listing.PageProperty(self, 'langlinks', 'll', - return_values=('lang', '*'), - **kwargs) + return listing.PageProperty(self, "langlinks", "ll", return_values=("lang", "*"), **kwargs) def links(self, namespace=None, generator=True, redirects=False): """List links to other pages from the current page. @@ -465,23 +465,35 @@ def links(self, namespace=None, generator=True, redirects=False): API doc: https://www.mediawiki.org/wiki/API:Links """ - prefix = listing.List.get_prefix('pl', generator) + prefix = listing.List.get_prefix("pl", generator) kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace)) if redirects: - kwargs['redirects'] = '1' + kwargs["redirects"] = "1" if generator: - return listing.PagePropertyGenerator(self, 'links', 'pl', **kwargs) + return listing.PagePropertyGenerator(self, "links", "pl", **kwargs) else: - return listing.PageProperty(self, 'links', 'pl', - return_values='title', **kwargs) - - def revisions(self, startid=None, endid=None, start=None, end=None, - dir='older', user=None, excludeuser=None, limit=None, - prop='ids|timestamp|flags|comment|user', - expandtemplates=False, section=None, - diffto=None, slots=None, uselang=None, max_items=None, - api_chunk_size=50): + return listing.PageProperty(self, "links", "pl", return_values="title", **kwargs) + + def revisions( + self, + startid=None, + endid=None, + start=None, + end=None, + dir="older", + user=None, + excludeuser=None, + limit=None, + prop="ids|timestamp|flags|comment|user", + expandtemplates=False, + section=None, + diffto=None, + slots=None, + uselang=None, + max_items=None, + api_chunk_size=50, + ): """List revisions of the current page. API doc: https://www.mediawiki.org/wiki/API:Revisions @@ -512,27 +524,35 @@ def revisions(self, startid=None, endid=None, start=None, end=None, listings.List: Revision iterator """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs( - 'rv', startid=startid, endid=endid, start=start, end=end, user=user, - excludeuser=excludeuser, diffto=diffto, slots=slots - )) + kwargs = dict( + listing.List.generate_kwargs( + "rv", + startid=startid, + endid=endid, + start=start, + end=end, + user=user, + excludeuser=excludeuser, + diffto=diffto, + slots=slots, + ) + ) - if self.site.version[:2] < (1, 32) and 'rvslots' in kwargs: + if self.site.version[:2] < (1, 32) and "rvslots" in kwargs: # https://github.com/mwclient/mwclient/issues/199 - del kwargs['rvslots'] + del kwargs["rvslots"] - kwargs['rvdir'] = dir - kwargs['rvprop'] = prop - kwargs['uselang'] = uselang + kwargs["rvdir"] = dir + kwargs["rvprop"] = prop + kwargs["uselang"] = uselang if expandtemplates: - kwargs['rvexpandtemplates'] = '1' + kwargs["rvexpandtemplates"] = "1" if section is not None: - kwargs['rvsection'] = section + kwargs["rvsection"] = section - return listing.RevisionsIterator(self, 'revisions', 'rv', - max_items=max_items, - api_chunk_size=api_chunk_size, - **kwargs) + return listing.RevisionsIterator( + self, "revisions", "rv", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + ) def templates(self, namespace=None, generator=True): """List templates used on the current page. @@ -540,11 +560,9 @@ def templates(self, namespace=None, generator=True): API doc: https://www.mediawiki.org/wiki/API:Templates """ - prefix = listing.List.get_prefix('tl', generator) + prefix = listing.List.get_prefix("tl", generator) kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace)) if generator: - return listing.PagePropertyGenerator(self, 'templates', prefix, - **kwargs) + return listing.PagePropertyGenerator(self, "templates", prefix, **kwargs) else: - return listing.PageProperty(self, 'templates', prefix, - return_values='title', **kwargs) + return listing.PageProperty(self, "templates", prefix, return_values="title", **kwargs) diff --git a/newapi/super/mwclient/sleep.py b/newapi/super/mwclient/sleep.py index d378f9e..73c9ab8 100644 --- a/newapi/super/mwclient/sleep.py +++ b/newapi/super/mwclient/sleep.py @@ -1,5 +1,6 @@ -import time import logging +import time + from .errors import MaximumRetriesExceeded log = logging.getLogger(__name__) @@ -25,6 +26,7 @@ class Sleepers: retry_timeout (int): The time to sleep for each past retry. callback (callable): A callable to be called on each retry. """ + def __init__(self, max_retries, retry_timeout, callback=lambda *x: None): self.max_retries = max_retries self.retry_timeout = retry_timeout @@ -59,6 +61,7 @@ class Sleeper: retry_timeout (int): The time to sleep for each past retry. callback (callable): A callable to be called on each retry. """ + def __init__(self, args, max_retries, retry_timeout, callback): self.args = args self.retries = 0 @@ -85,6 +88,6 @@ def sleep(self, min_time=0): if timeout < min_time: timeout = min_time - print(f'mwclient/sleep.py: Sleeping for {timeout} seconds') + print(f"mwclient/sleep.py: Sleeping for {timeout} seconds") # time.sleep(timeout) diff --git a/newapi/super/mwclient/util.py b/newapi/super/mwclient/util.py index 9f02705..fe6d0ec 100644 --- a/newapi/super/mwclient/util.py +++ b/newapi/super/mwclient/util.py @@ -1,5 +1,5 @@ -import time import io +import time import warnings @@ -12,9 +12,9 @@ def parse_timestamp(t): Returns: time.struct_time: A timestamp. """ - if t is None or t == '0000-00-00T00:00:00Z': + if t is None or t == "0000-00-00T00:00:00Z": return time.struct_time((0, 0, 0, 0, 0, 0, 0, 0, 0)) - return time.strptime(t, '%Y-%m-%dT%H:%M:%SZ') + return time.strptime(t, "%Y-%m-%dT%H:%M:%SZ") def read_in_chunks(stream, chunk_size): @@ -38,7 +38,7 @@ def handle_limit(limit, max_items, api_chunk_size): warnings.warn( "limit and api_chunk_size both specified, this is not supported! limit " "is deprecated, will use value of api_chunk_size", - DeprecationWarning + DeprecationWarning, ) else: warnings.warn( @@ -46,7 +46,7 @@ def handle_limit(limit, max_items, api_chunk_size): "api_chunk_size to set the number of items retrieved from the API at " "once, and/or max_items to limit the total number of items that will be " "yielded", - DeprecationWarning + DeprecationWarning, ) api_chunk_size = limit return (max_items, api_chunk_size) diff --git a/newapi/super/params_help.py b/newapi/super/params_help.py index 0abfb6d..d666b7a 100644 --- a/newapi/super/params_help.py +++ b/newapi/super/params_help.py @@ -3,8 +3,10 @@ from .super.params_help import PARAMS_HELPS """ -import sys + import json +import sys + from ..api_utils.except_err import exception_err @@ -18,7 +20,13 @@ def __init__(self) -> None: # pass def params_w(self, params) -> dict: - if self.family == "wikipedia" and self.lang == "ar" and params.get("summary") and self.username.find("bot") == -1 and "ibrahemsummary" not in sys.argv: + if ( + self.family == "wikipedia" + and self.lang == "ar" + and params.get("summary") + and self.username.find("bot") == -1 + and "ibrahemsummary" not in sys.argv + ): params["summary"] = "" self.Bot_or_himo = 1 if "bot" in self.username else 0 @@ -30,7 +38,11 @@ def params_w(self, params) -> dict: params["minor"] = self.Bot_or_himo if self.family != "toolforge": - if params["action"] in ["edit", "create", "upload", "delete", "move"] or params["action"].startswith("wb") or self.family == "wikidata": + if ( + params["action"] in ["edit", "create", "upload", "delete", "move"] + or params["action"].startswith("wb") + or self.family == "wikidata" + ): if "nologin" not in sys.argv and self.username: params["assertuser"] = self.username diff --git a/newapi/super/super_login.py b/newapi/super/super_login.py index 6fa3126..893dce6 100644 --- a/newapi/super/super_login.py +++ b/newapi/super/super_login.py @@ -19,9 +19,9 @@ import urllib.parse from ..api_utils import printe -from .handel_errors import HANDEL_ERRORS from ..api_utils.except_err import warn_err from ..api_utils.user_agent import default_user_agent +from .handel_errors import HANDEL_ERRORS # if "nomwclient" in sys.argv: # from .bot import LOGIN_HELPS @@ -37,6 +37,7 @@ ar_lag = {1: 3} urls_prints = {"all": 0} + class Login(LOGIN_HELPS, HANDEL_ERRORS): """ Represents a login session for a wiki. @@ -79,7 +80,11 @@ def p_url(self, params): no_url = ["lgpassword", "format"] no_remove = ["titles", "title"] # --- - pams2 = {k: v[:100] if isinstance(v, str) and len(v) > 100 and k not in no_remove else v for k, v in params.items() if k not in no_url} + pams2 = { + k: v[:100] if isinstance(v, str) and len(v) > 100 and k not in no_remove else v + for k, v in params.items() + if k not in no_url + } # --- self.url_o_print = f"{self.endpoint}?{urllib.parse.urlencode(pams2)}".replace("&format=json", "") # --- @@ -126,7 +131,11 @@ def filter_params(self, params): if self.family == "nccommons" and params.get("bot"): del params["bot"] - if "workibrahem" in sys.argv and "ibrahemsummary" not in sys.argv and params.get("summary", "").find("بوت:") != -1: + if ( + "workibrahem" in sys.argv + and "ibrahemsummary" not in sys.argv + and params.get("summary", "").find("بوت:") != -1 + ): params["summary"] = "" if params["action"] in ["query"]: @@ -226,7 +235,9 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N return data - def post_continue(self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None): + def post_continue( + self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + ): # --- printe.test_print("_______________________") printe.test_print(f"post_continue, start. {action=}, {_p_=}") diff --git a/tests/TestALL_APIS.py b/tests/TestALL_APIS.py index 7c6525e..ac6ecd6 100644 --- a/tests/TestALL_APIS.py +++ b/tests/TestALL_APIS.py @@ -1,32 +1,35 @@ -import pytest from unittest.mock import MagicMock, patch + +import pytest from newapi import ALL_APIS @pytest.fixture def mock_dependencies(): - with patch('newapi.pages_bots.all_apis.Login') as mock_login, \ - patch('newapi.pages_bots.all_apis.super_page.MainPage') as mock_main_page, \ - patch('newapi.pages_bots.all_apis.catdepth_new.subcatquery') as mock_subcatquery, \ - patch('newapi.pages_bots.all_apis.bot_api.NEW_API') as mock_new_api, \ - patch('newapi.pages_bots.all_apis.printe') as mock_printe: + with ( + patch("newapi.pages_bots.all_apis.Login") as mock_login, + patch("newapi.pages_bots.all_apis.super_page.MainPage") as mock_main_page, + patch("newapi.pages_bots.all_apis.catdepth_new.subcatquery") as mock_subcatquery, + patch("newapi.pages_bots.all_apis.bot_api.NEW_API") as mock_new_api, + patch("newapi.pages_bots.all_apis.printe") as mock_printe, + ): # Setup mock login instance mock_login_instance = MagicMock() mock_login.return_value = mock_login_instance yield { - 'Login': mock_login, - 'LoginInstance': mock_login_instance, - 'MainPage': mock_main_page, - 'subcatquery': mock_subcatquery, - 'NEW_API': mock_new_api, - 'printe': mock_printe + "Login": mock_login, + "LoginInstance": mock_login_instance, + "MainPage": mock_main_page, + "subcatquery": mock_subcatquery, + "NEW_API": mock_new_api, + "printe": mock_printe, } def test_all_apis_init(mock_dependencies): - lang, family, username, password = 'en', 'wikipedia', 'user', 'pass' + lang, family, username, password = "en", "wikipedia", "user", "pass" api = ALL_APIS(lang, family, username, password) assert api.lang == lang @@ -35,47 +38,47 @@ def test_all_apis_init(mock_dependencies): assert api.password == password # Verify login was called - mock_dependencies['Login'].assert_called_once_with(lang, family=family) - mock_dependencies['LoginInstance'].add_users.assert_called_once() + mock_dependencies["Login"].assert_called_once_with(lang, family=family) + mock_dependencies["LoginInstance"].add_users.assert_called_once() # Verify printe.output was called - mock_dependencies['printe'].output.assert_called() + mock_dependencies["printe"].output.assert_called() def test_all_apis_main_page(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') - title = 'Test Page' + api = ALL_APIS("en", "wikipedia", "user", "pass") + title = "Test Page" api.MainPage(title) - mock_dependencies['MainPage'].assert_called_once_with( - mock_dependencies['LoginInstance'], title, 'en', family='wikipedia' + mock_dependencies["MainPage"].assert_called_once_with( + mock_dependencies["LoginInstance"], title, "en", family="wikipedia" ) def test_all_apis_cat_depth(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') - title = 'Category:Test' + api = ALL_APIS("en", "wikipedia", "user", "pass") + title = "Category:Test" api.CatDepth(title, depth=2) - mock_dependencies['subcatquery'].assert_called_once_with( - mock_dependencies['LoginInstance'], title, sitecode='en', family='wikipedia', depth=2 + mock_dependencies["subcatquery"].assert_called_once_with( + mock_dependencies["LoginInstance"], title, sitecode="en", family="wikipedia", depth=2 ) def test_all_apis_new_api(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') + api = ALL_APIS("en", "wikipedia", "user", "pass") api.NEW_API() - mock_dependencies['NEW_API'].assert_called_once_with( - mock_dependencies['LoginInstance'], lang='en', family='wikipedia' + mock_dependencies["NEW_API"].assert_called_once_with( + mock_dependencies["LoginInstance"], lang="en", family="wikipedia" ) def test_login_lru_cache(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') + api = ALL_APIS("en", "wikipedia", "user", "pass") # Call _login multiple times api._login() @@ -83,4 +86,4 @@ def test_login_lru_cache(mock_dependencies): # Login should only be instantiated once due to lru_cache # Note: __init__ calls _login once, so total calls should be 1 - assert mock_dependencies['Login'].call_count == 1 + assert mock_dependencies["Login"].call_count == 1 diff --git a/tests/TestAuthentication.py b/tests/TestAuthentication.py index fe47715..d527062 100644 --- a/tests/TestAuthentication.py +++ b/tests/TestAuthentication.py @@ -2,13 +2,11 @@ from newapi import useraccount from newapi.super import super_login + class TestAuthentication: @pytest.fixture def user_credentials(self): - return { - "username": useraccount.username, - "password": useraccount.password - } + return {"username": useraccount.username, "password": useraccount.password} @pytest.fixture def login_client(self, user_credentials): @@ -18,21 +16,14 @@ def login_client(self, user_credentials): def test_successful_login(self, login_client): """Test successful authentication""" - params = { - "action": "query", - "titles": "Main Page", - "format": "json" - } + params = {"action": "query", "titles": "Main Page", "format": "json"} response = login_client.post(params, Type="post", addtoken=False) assert response is not None assert len(response) > 0 def test_invalid_credentials(self): """Test authentication with invalid credentials""" - invalid_creds = { - "username": useraccount.username, - "password": f"{useraccount.password}213" - } + invalid_creds = {"username": useraccount.username, "password": f"{useraccount.password}213"} bot = super_login.Login("en", family="wikipedia") # Test should handle authentication failure gracefully login_result = bot.Log_to_wiki() diff --git a/tests/TestLiteDB.py b/tests/TestLiteDB.py index dd47f27..64657e1 100644 --- a/tests/TestLiteDB.py +++ b/tests/TestLiteDB.py @@ -1,13 +1,15 @@ -import pytest -import tempfile import os +import tempfile + +import pytest from newapi.DB_bots.db_bot import LiteDB + class TestLiteDB: @pytest.fixture def temp_db(self): """Create temporary database for testing""" - fd, path = tempfile.mkstemp(suffix='.db') + fd, path = tempfile.mkstemp(suffix=".db") os.close(fd) db = LiteDB(path) yield db diff --git a/tests/TestMainPage.py b/tests/TestMainPage.py index c7269ff..6c6cd8d 100644 --- a/tests/TestMainPage.py +++ b/tests/TestMainPage.py @@ -1,10 +1,11 @@ import pytest from newapi.page import MainPage + class TestMainPage: @pytest.fixture def test_page(self): - return MainPage("User:Mr. Ibrahem/sandbox", 'en') + return MainPage("User:Mr. Ibrahem/sandbox", "en") @pytest.fixture def arabic_page(self): @@ -29,7 +30,7 @@ def test_get_text(self, arabic_page): def test_nonexistent_page(self): """Test behavior with non-existent page""" - page = MainPage("NonExistentPage12345", 'en') + page = MainPage("NonExistentPage12345", "en") assert page.exists() is False assert isinstance(page.get_text(), str) # Should handle gracefully @@ -41,7 +42,7 @@ def test_empty_page_content(self): def test_page_without_edit_permission(self): """Test page where user cannot edit""" # Test with a protected page or mock this scenario - page = MainPage("الصفحة الرئيسة", 'ar') + page = MainPage("الصفحة الرئيسة", "ar") assert page.can_edit() is False def test_page_title_validation(self): diff --git a/tests/TestNewAPI.py b/tests/TestNewAPI.py index 6abeb5c..657b63b 100644 --- a/tests/TestNewAPI.py +++ b/tests/TestNewAPI.py @@ -9,10 +9,7 @@ def api_client(self): def test_find_pages_exists(self, api_client): """Test Find_pages_exists_or_not method""" - result = api_client.Find_pages_exists_or_not( - ["Thyrotropin alfa", "Thiamine"], - get_redirect=True - ) + result = api_client.Find_pages_exists_or_not(["Thyrotropin alfa", "Thiamine"], get_redirect=True) assert result is not None assert isinstance(result, (dict, list)) diff --git a/tests/conftest.py b/tests/conftest.py index 6c048e4..1b58457 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,19 @@ -import pytest -import sys import os +import sys + +import pytest # إضافة مسار المشروع إلى sys.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + @pytest.fixture(scope="session") def api_credentials(): """بيانات اعتماد مشتركة للاختبارات""" from newapi import useraccount - return { - "username": useraccount.username, - "password": useraccount.password - } + + return {"username": useraccount.username, "password": useraccount.password} + @pytest.fixture def temp_test_page(): diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py index 0757eda..82f09d8 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py @@ -4,14 +4,15 @@ Tests cover all scenarios including template restrictions, caching behavior, and special template handling (nobots, bots). """ + import sys -import pytest +import pytest from newapi.api_utils.bot_edit.bot_edit_by_templates import ( - is_bot_edit_allowed, Bot_Cache, - stop_edit_temps, edit_username, + is_bot_edit_allowed, + stop_edit_temps, ) @@ -369,10 +370,6 @@ def test_different_botjob_not_affected_by_specific_template(self, original_argv) sys.argv = ["script"] text = "{{لا للتعريب}}" # Should block for 'تعريب' botjob - assert not is_bot_edit_allowed( - text=text, title_page="Test", botjob="تعريب" - ) + assert not is_bot_edit_allowed(text=text, title_page="Test", botjob="تعريب") # Should allow for 'all' botjob (not in 'all' stop list) - assert is_bot_edit_allowed( - text=text, title_page="Test2", botjob="all" - ) + assert is_bot_edit_allowed(text=text, title_page="Test2", botjob="all") diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py index b651959..1a12a11 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py @@ -8,26 +8,26 @@ - Edge cases and special conditions """ -import pytest -from unittest.mock import patch, MagicMock import sys +from unittest.mock import MagicMock, patch +import pytest from newapi.api_utils.bot_edit.bot_edit_by_templates import ( - is_bot_edit_allowed, Bot_Cache, + is_bot_edit_allowed, stop_edit_temps, ) - # ==================== Fixtures ==================== + @pytest.fixture(autouse=True) def reset_environment(): """Reset environment before and after each test.""" # Setup Bot_Cache.clear() original_argv = sys.argv.copy() - sys.argv = ['test'] + sys.argv = ["test"] yield @@ -39,13 +39,14 @@ def reset_environment(): @pytest.fixture def mock_wtp(): """Provide a mocked wikitextparser.""" - with patch('newapi.api_utils.bot_edit.bot_edit_by_templates.wtp') as mock: + with patch("newapi.api_utils.bot_edit.bot_edit_by_templates.wtp") as mock: yield mock @pytest.fixture def create_mock_template(): """Factory fixture for creating mock templates.""" + def _create_template(name, arguments=None): mock_template = MagicMock() mock_template.normal_name.return_value = name @@ -70,12 +71,13 @@ def _create_template(name, arguments=None): @pytest.fixture def setup_parser(mock_wtp, create_mock_template): """Factory fixture for setting up parser with templates.""" + def _setup(templates_config): templates = [] for config in templates_config: if isinstance(config, dict): - name = config.get('name') - arguments = config.get('arguments') + name = config.get("name") + arguments = config.get("arguments") templates.append(create_mock_template(name, arguments)) else: # If just a string, create template with that name @@ -92,6 +94,7 @@ def _setup(templates_config): # ==================== Basic Functionality Tests ==================== + class TestBasicFunctionality: """Test basic functionality of is_bot_edit_allowed.""" @@ -116,17 +119,14 @@ def test_default_botjob_parameter(self): def test_fixref_cat_stub_tempcat_portal_defaults_to_all(self): """Test that combined botjob string defaults to 'all'.""" text = "Plain text" - result = is_bot_edit_allowed( - text=text, - title_page="Test Page", - botjob="fixref|cat|stub|tempcat|portal" - ) + result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="fixref|cat|stub|tempcat|portal") assert result is True assert "all" in Bot_Cache # ==================== Cache Tests ==================== + class TestCacheBehavior: """Test cache behavior and management.""" @@ -186,6 +186,7 @@ def test_cache_persists_across_calls(self): assert first_cache_value == second_cache_value + # ==================== Nobots Template Tests ==================== @@ -194,7 +195,7 @@ class TestNobotsTemplate: def test_nobots_without_params_denies_edit(self, setup_parser): """Test that {{nobots}} without parameters denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) text = "{{nobots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -202,7 +203,7 @@ def test_nobots_without_params_denies_edit(self, setup_parser): def test_nobots_with_all_denies_edit(self, setup_parser): """Test that {{nobots|1=all}} denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'all'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "all"}}]) text = "{{nobots|1=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -210,7 +211,7 @@ def test_nobots_with_all_denies_edit(self, setup_parser): def test_nobots_with_specific_bot_denies_edit(self, setup_parser): """Test that {{nobots|1=Mr.Ibrahembot}} denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'Mr.Ibrahembot'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "Mr.Ibrahembot"}}]) text = "{{nobots|1=Mr.Ibrahembot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -218,7 +219,7 @@ def test_nobots_with_specific_bot_denies_edit(self, setup_parser): def test_nobots_with_bot_list_including_our_bot_denies(self, setup_parser): """Test that {{nobots|1=Bot1,Mr.Ibrahembot,Bot2}} denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'Bot1,Mr.Ibrahembot,Bot2'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "Bot1,Mr.Ibrahembot,Bot2"}}]) text = "{{nobots|1=Bot1,Mr.Ibrahembot,Bot2}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -226,7 +227,7 @@ def test_nobots_with_bot_list_including_our_bot_denies(self, setup_parser): def test_nobots_with_other_bots_allows_edit(self, setup_parser): """Test that {{nobots|1=OtherBot}} allows editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'OtherBot,AnotherBot'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "OtherBot,AnotherBot"}}]) text = "{{nobots|1=OtherBot,AnotherBot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -234,7 +235,7 @@ def test_nobots_with_other_bots_allows_edit(self, setup_parser): def test_nobots_case_insensitive(self, setup_parser): """Test that nobots template matching is case insensitive.""" - setup_parser([{'name': 'NoBots', 'arguments': None}]) + setup_parser([{"name": "NoBots", "arguments": None}]) text = "{{NoBots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -242,7 +243,7 @@ def test_nobots_case_insensitive(self, setup_parser): def test_nobots_with_whitespace_in_bot_names(self, setup_parser): """Test handling of whitespace in bot name lists.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': ' Bot1 , Mr.Ibrahembot , Bot2 '}}]) + setup_parser([{"name": "nobots", "arguments": {"1": " Bot1 , Mr.Ibrahembot , Bot2 "}}]) text = "{{nobots|1= Bot1 , Mr.Ibrahembot , Bot2 }}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -251,12 +252,13 @@ def test_nobots_with_whitespace_in_bot_names(self, setup_parser): # ==================== Bots Template Tests ==================== + class TestBotsTemplate: """Test bots template handling.""" def test_bots_without_params_denies_edit(self, setup_parser): """Test that {{bots}} without parameters denies editing.""" - setup_parser([{'name': 'bots', 'arguments': None}]) + setup_parser([{"name": "bots", "arguments": None}]) text = "{{bots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -264,7 +266,7 @@ def test_bots_without_params_denies_edit(self, setup_parser): def test_bots_allow_all_allows_edit(self, setup_parser): """Test that {{bots|allow=all}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'all'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "all"}}]) text = "{{bots|allow=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -272,7 +274,7 @@ def test_bots_allow_all_allows_edit(self, setup_parser): def test_bots_allow_specific_bot_allows_edit(self, setup_parser): """Test that {{bots|allow=Mr.Ibrahembot}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'Mr.Ibrahembot'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "Mr.Ibrahembot"}}]) text = "{{bots|allow=Mr.Ibrahembot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -280,7 +282,7 @@ def test_bots_allow_specific_bot_allows_edit(self, setup_parser): def test_bots_allow_bot_list_including_our_bot_allows(self, setup_parser): """Test that {{bots|allow=Bot1,Mr.Ibrahembot,Bot2}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'Bot1,Mr.Ibrahembot,Bot2'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "Bot1,Mr.Ibrahembot,Bot2"}}]) text = "{{bots|allow=Bot1,Mr.Ibrahembot,Bot2}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -288,7 +290,7 @@ def test_bots_allow_bot_list_including_our_bot_allows(self, setup_parser): def test_bots_allow_none_denies_edit(self, setup_parser): """Test that {{bots|allow=none}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'none'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "none"}}]) text = "{{bots|allow=none}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -296,7 +298,7 @@ def test_bots_allow_none_denies_edit(self, setup_parser): def test_bots_allow_other_bots_denies_edit(self, setup_parser): """Test that {{bots|allow=OtherBot}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'OtherBot,AnotherBot'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "OtherBot,AnotherBot"}}]) text = "{{bots|allow=OtherBot,AnotherBot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -304,7 +306,7 @@ def test_bots_allow_other_bots_denies_edit(self, setup_parser): def test_bots_deny_all_denies_edit(self, setup_parser): """Test that {{bots|deny=all}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': 'all'}}]) + setup_parser([{"name": "bots", "arguments": {"deny": "all"}}]) text = "{{bots|deny=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -312,7 +314,7 @@ def test_bots_deny_all_denies_edit(self, setup_parser): def test_bots_deny_specific_bot_denies_edit(self, setup_parser): """Test that {{bots|deny=Mr.Ibrahembot}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': 'Mr.Ibrahembot'}}]) + setup_parser([{"name": "bots", "arguments": {"deny": "Mr.Ibrahembot"}}]) text = "{{bots|deny=Mr.Ibrahembot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -320,7 +322,7 @@ def test_bots_deny_specific_bot_denies_edit(self, setup_parser): def test_bots_deny_other_bots_allows_edit(self, setup_parser): """Test that {{bots|deny=OtherBot}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': 'OtherBot,AnotherBot'}}]) + setup_parser([{"name": "bots", "arguments": {"deny": "OtherBot,AnotherBot"}}]) text = "{{bots|deny=OtherBot,AnotherBot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -328,7 +330,7 @@ def test_bots_deny_other_bots_allows_edit(self, setup_parser): def test_bots_case_insensitive(self, setup_parser): """Test that bots template matching is case insensitive.""" - setup_parser([{'name': 'Bots', 'arguments': {'allow': 'all'}}]) + setup_parser([{"name": "Bots", "arguments": {"allow": "all"}}]) text = "{{Bots|allow=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -337,13 +339,14 @@ def test_bots_case_insensitive(self, setup_parser): # ==================== Stop Edit Templates Tests ==================== + class TestStopEditTemplates: """Test stop edit templates handling.""" @pytest.mark.parametrize("template_name", stop_edit_temps["all"]) def test_global_stop_templates_deny_edit(self, template_name, setup_parser): """Test that global stop templates deny editing.""" - setup_parser([{'name': template_name, 'arguments': None}]) + setup_parser([{"name": template_name, "arguments": None}]) text = f"{{{{{template_name}}}}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -352,7 +355,7 @@ def test_global_stop_templates_deny_edit(self, template_name, setup_parser): def test_specific_botjob_stop_template_denies_edit(self, setup_parser): """Test that job-specific stop templates deny editing.""" # Test تعريب job with its specific template - setup_parser([{'name': 'لا للتعريب', 'arguments': None}]) + setup_parser([{"name": "لا للتعريب", "arguments": None}]) text = "{{لا للتعريب}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="تعريب") @@ -361,22 +364,20 @@ def test_specific_botjob_stop_template_denies_edit(self, setup_parser): def test_stop_template_for_different_botjob_allows_edit(self, setup_parser): """Test that stop templates for different bot jobs allow editing.""" # Template for تعريب job, but we're running cat job - setup_parser([{'name': 'لا للتعريب', 'arguments': None}]) + setup_parser([{"name": "لا للتعريب", "arguments": None}]) text = "{{لا للتعريب}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="cat") assert result is True - @pytest.mark.parametrize("botjob,template_list", [ - (job, templates) - for job, templates in stop_edit_temps.items() - if job != "all" - ]) + @pytest.mark.parametrize( + "botjob,template_list", [(job, templates) for job, templates in stop_edit_temps.items() if job != "all"] + ) def test_all_stop_templates_for_each_botjob(self, botjob, template_list, setup_parser): """Test all stop templates for each specific bot job.""" for template_name in template_list: Bot_Cache.clear() - setup_parser([{'name': template_name, 'arguments': None}]) + setup_parser([{"name": template_name, "arguments": None}]) text = f"{{{{{template_name}}}}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob=botjob) @@ -385,15 +386,13 @@ def test_all_stop_templates_for_each_botjob(self, botjob, template_list, setup_p # ==================== Multiple Templates Tests ==================== + class TestMultipleTemplates: """Test handling of multiple templates.""" def test_multiple_templates_first_restricting_denies(self, setup_parser): """Test that first restricting template denies editing.""" - setup_parser([ - {'name': 'nobots', 'arguments': None}, - {'name': 'some_other_template', 'arguments': None} - ]) + setup_parser([{"name": "nobots", "arguments": None}, {"name": "some_other_template", "arguments": None}]) text = "{{nobots}} {{some_other_template}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -401,10 +400,7 @@ def test_multiple_templates_first_restricting_denies(self, setup_parser): def test_multiple_non_restricting_templates_allows(self, setup_parser): """Test that multiple non-restricting templates allow editing.""" - setup_parser([ - {'name': 'infobox', 'arguments': None}, - {'name': 'citation', 'arguments': None} - ]) + setup_parser([{"name": "infobox", "arguments": None}, {"name": "citation", "arguments": None}]) text = "{{infobox}} {{citation}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -413,12 +409,13 @@ def test_multiple_non_restricting_templates_allows(self, setup_parser): # ==================== Edge Cases Tests ==================== + class TestEdgeCases: """Test edge cases and special conditions.""" def test_empty_template_parameters(self, setup_parser): """Test handling of templates with empty parameter values.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': ''}}]) + setup_parser([{"name": "nobots", "arguments": {"1": ""}}]) text = "{{nobots|1=}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -427,10 +424,7 @@ def test_empty_template_parameters(self, setup_parser): def test_template_with_multiple_parameters(self, setup_parser): """Test template with multiple parameters.""" - setup_parser([{ - 'name': 'bots', - 'arguments': {'allow': 'all', 'other_param': 'some_value'} - }]) + setup_parser([{"name": "bots", "arguments": {"allow": "all", "other_param": "some_value"}}]) text = "{{bots|allow=all|other_param=some_value}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -438,10 +432,7 @@ def test_template_with_multiple_parameters(self, setup_parser): def test_parameter_filtering_empty_values(self, setup_parser): """Test that parameters with empty values are filtered out.""" - setup_parser([{ - 'name': 'bots', - 'arguments': {'allow': 'all', 'empty_param': ''} - }]) + setup_parser([{"name": "bots", "arguments": {"allow": "all", "empty_param": ""}}]) text = "{{bots|allow=all|empty_param=}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -450,14 +441,18 @@ def test_parameter_filtering_empty_values(self, setup_parser): # ==================== Parametrized Test Collections ==================== + class TestParametrizedScenarios: """Parametrized tests for various scenarios.""" - @pytest.mark.parametrize("text,expected", [ - ("", True), - ("Plain text", True), - ("Some content without templates", True), - ]) + @pytest.mark.parametrize( + "text,expected", + [ + ("", True), + ("Plain text", True), + ("Some content without templates", True), + ], + ) def test_non_template_text_allows_editing(self, text, expected): """Test that non-template text allows editing.""" result = is_bot_edit_allowed(text=text, title_page="Test", botjob="all") @@ -470,44 +465,53 @@ def test_cache_initialized_for_different_botjobs(self, botjob): expected_job = "all" if botjob in ["", "fixref|cat|stub|tempcat|portal"] else botjob assert expected_job in Bot_Cache - @pytest.mark.parametrize("bot_list,should_allow", [ - ("OtherBot", True), - ("Bot1,Bot2,Bot3", True), - ("Mr.Ibrahembot", False), - ("Bot1,Mr.Ibrahembot", False), - ("all", False), - ]) + @pytest.mark.parametrize( + "bot_list,should_allow", + [ + ("OtherBot", True), + ("Bot1,Bot2,Bot3", True), + ("Mr.Ibrahembot", False), + ("Bot1,Mr.Ibrahembot", False), + ("all", False), + ], + ) def test_nobots_with_various_bot_lists(self, bot_list, should_allow, setup_parser): """Test nobots template with various bot lists.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': bot_list}}]) + setup_parser([{"name": "nobots", "arguments": {"1": bot_list}}]) result = is_bot_edit_allowed(text="{{nobots}}", title_page="Test", botjob="all") assert result == should_allow - @pytest.mark.parametrize("allow_list,should_allow", [ - ("all", True), - ("Mr.Ibrahembot", True), - ("Bot1,Mr.Ibrahembot,Bot2", True), - ("none", False), - ("OtherBot", False), - ]) + @pytest.mark.parametrize( + "allow_list,should_allow", + [ + ("all", True), + ("Mr.Ibrahembot", True), + ("Bot1,Mr.Ibrahembot,Bot2", True), + ("none", False), + ("OtherBot", False), + ], + ) def test_bots_allow_with_various_lists(self, allow_list, should_allow, setup_parser): """Test bots allow parameter with various lists.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': allow_list}}]) + setup_parser([{"name": "bots", "arguments": {"allow": allow_list}}]) result = is_bot_edit_allowed(text="{{bots}}", title_page="Test", botjob="all") assert result == should_allow - @pytest.mark.parametrize("deny_list,should_allow", [ - ("all", False), - ("Mr.Ibrahembot", False), - ("Bot1,Mr.Ibrahembot,Bot2", False), - ("OtherBot", True), - ("Bot1,Bot2", True), - ]) + @pytest.mark.parametrize( + "deny_list,should_allow", + [ + ("all", False), + ("Mr.Ibrahembot", False), + ("Bot1,Mr.Ibrahembot,Bot2", False), + ("OtherBot", True), + ("Bot1,Bot2", True), + ], + ) def test_bots_deny_with_various_lists(self, deny_list, should_allow, setup_parser): """Test bots deny parameter with various lists.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': deny_list}}]) + setup_parser([{"name": "bots", "arguments": {"deny": deny_list}}]) result = is_bot_edit_allowed(text="{{bots}}", title_page="Test", botjob="all") assert result == should_allow @@ -515,27 +519,20 @@ def test_bots_deny_with_various_lists(self, deny_list, should_allow, setup_parse # ==================== Integration Tests ==================== + class TestIntegration: """Integration tests for complete workflows.""" def test_complete_workflow_allowed(self, setup_parser): """Test complete workflow where editing is allowed.""" # Setup non-restricting template - setup_parser([{'name': 'infobox', 'arguments': None}]) + setup_parser([{"name": "infobox", "arguments": None}]) # First call - process and cache - result1 = is_bot_edit_allowed( - text="{{infobox}}", - title_page="Article", - botjob="all" - ) + result1 = is_bot_edit_allowed(text="{{infobox}}", title_page="Article", botjob="all") # Second call - use cache - result2 = is_bot_edit_allowed( - text="{{infobox}}", - title_page="Article", - botjob="all" - ) + result2 = is_bot_edit_allowed(text="{{infobox}}", title_page="Article", botjob="all") assert result1 is True assert result2 is True @@ -545,21 +542,13 @@ def test_complete_workflow_allowed(self, setup_parser): def test_complete_workflow_denied(self, setup_parser): """Test complete workflow where editing is denied.""" # Setup restricting template - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) # First call - process and cache - result1 = is_bot_edit_allowed( - text="{{nobots}}", - title_page="Article", - botjob="all" - ) + result1 = is_bot_edit_allowed(text="{{nobots}}", title_page="Article", botjob="all") # Second call - use cache - result2 = is_bot_edit_allowed( - text="{{nobots}}", - title_page="Article", - botjob="all" - ) + result2 = is_bot_edit_allowed(text="{{nobots}}", title_page="Article", botjob="all") assert result1 is False assert result2 is False @@ -569,11 +558,11 @@ def test_complete_workflow_denied(self, setup_parser): def test_different_pages_different_results(self, setup_parser): """Test that different pages can have different results.""" # Page 1 - allowed - setup_parser([{'name': 'infobox', 'arguments': None}]) + setup_parser([{"name": "infobox", "arguments": None}]) result1 = is_bot_edit_allowed(text="{{infobox}}", title_page="Page1", botjob="all") # Page 2 - denied - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) result2 = is_bot_edit_allowed(text="{{nobots}}", title_page="Page2", botjob="all") assert result1 is True @@ -584,29 +573,30 @@ def test_different_pages_different_results(self, setup_parser): # ==================== Performance and Special Cases ==================== + class TestSpecialCases: """Test special cases and boundary conditions.""" def test_very_long_bot_list(self, setup_parser): """Test handling of very long bot lists.""" long_list = ",".join([f"Bot{i}" for i in range(100)]) - setup_parser([{'name': 'nobots', 'arguments': {'1': long_list}}]) + setup_parser([{"name": "nobots", "arguments": {"1": long_list}}]) result = is_bot_edit_allowed(text="{{nobots}}", title_page="Test", botjob="all") assert result is True # Our bot not in the list def test_unicode_bot_names(self, setup_parser): """Test handling of unicode characters in bot names.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'بوت1,بوت2'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "بوت1,بوت2"}}]) result = is_bot_edit_allowed(text="{{nobots}}", title_page="Test", botjob="all") assert result is True # Our bot not in the list def test_mixed_case_template_names(self, setup_parser): """Test that template name matching works with mixed case.""" - for name in ['nobots', 'Nobots', 'NOBOTS', 'nObOtS']: + for name in ["nobots", "Nobots", "NOBOTS", "nObOtS"]: Bot_Cache.clear() - setup_parser([{'name': name, 'arguments': None}]) + setup_parser([{"name": name, "arguments": None}]) result = is_bot_edit_allowed(text=f"{{{{{name}}}}}", title_page="Test", botjob="all") assert result is False, f"Should deny for template name: {name}" @@ -614,6 +604,7 @@ def test_mixed_case_template_names(self, setup_parser): # ==================== Pytest Marks and Markers ==================== + class TestSmokeTests: """Quick smoke tests for CI/CD.""" @@ -623,7 +614,7 @@ def test_basic_allow(self): def test_basic_deny(self, setup_parser): """Basic deny scenario.""" - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) assert is_bot_edit_allowed("{{nobots}}", "Test", "all") is False def test_cache_works(self): diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py index f964873..48e5cb3 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py @@ -1,13 +1,12 @@ -""" -""" +""" """ -import pytest import sys -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch +import pytest from newapi.api_utils.bot_edit.bot_edit_by_templates import ( - is_bot_edit_allowed, Bot_Cache, + is_bot_edit_allowed, ) # ==================== Fixtures ==================== @@ -27,7 +26,7 @@ def reset_environment(): # Setup Bot_Cache.clear() original_argv = sys.argv.copy() - sys.argv = ['test'] + sys.argv = ["test"] yield @@ -39,13 +38,14 @@ def reset_environment(): @pytest.fixture def mock_wtp(): """Provide a mocked wikitextparser.""" - with patch('newapi.api_utils.bot_edit.bot_edit_by_templates.wtp') as mock: + with patch("newapi.api_utils.bot_edit.bot_edit_by_templates.wtp") as mock: yield mock @pytest.fixture def create_mock_template(): """Factory fixture for creating mock templates.""" + def _create_template(name, arguments=None): mock_template = MagicMock() mock_template.normal_name.return_value = name @@ -70,12 +70,13 @@ def _create_template(name, arguments=None): @pytest.fixture def setup_parser(mock_wtp, create_mock_template): """Factory fixture for setting up parser with templates.""" + def _setup(templates_config): templates = [] for config in templates_config: if isinstance(config, dict): - name = config.get('name') - arguments = config.get('arguments') + name = config.get("name") + arguments = config.get("arguments") templates.append(create_mock_template(name, arguments)) else: # If just a string, create template with that name @@ -92,6 +93,7 @@ def _setup(templates_config): # ==================== Command Line Bypass Tests ==================== + class TestCommandLineBypass: """Test command line argument bypass functionality.""" @@ -99,7 +101,7 @@ class TestCommandLineBypass: def test_argv_bypasses_all_checks(self, argv_value, setup_parser): """Test that specific argv values bypass all restrictions.""" sys.argv.append(argv_value) - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) text = "{{nobots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py b/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py index 2464ca5..594d828 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py @@ -1,8 +1,8 @@ -""" -""" +""" """ + import sys -import pytest +import pytest from newapi.api_utils.bot_edit.bot_edit_by_time import ( check_create_time, check_last_edit_time, From 71c034333a5ae8481a5bdde8be90ce6b4d32ed84 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Sun, 1 Feb 2026 03:14:34 +0300 Subject: [PATCH 09/11] Remove __main__ test blocks from utility modules Deleted standalone test and debug code under '__main__' in except_err.py, printe.py, txtlib.py, pformat.py, and ar_err.py to clean up production code and prevent accidental execution of test routines. --- newapi/api_utils/except_err.py | 8 -------- newapi/api_utils/printe.py | 13 ------------ newapi/api_utils/txtlib.py | 32 ------------------------------ newapi/pformat.py | 36 ---------------------------------- newapi/super/S_Page/ar_err.py | 5 ----- 5 files changed, 94 deletions(-) diff --git a/newapi/api_utils/except_err.py b/newapi/api_utils/except_err.py index 24784bc..ecbf658 100644 --- a/newapi/api_utils/except_err.py +++ b/newapi/api_utils/except_err.py @@ -50,11 +50,3 @@ def exception_err(e, text=""): # --- printe.warn("CRITICAL:") # printe.info("====") - - -if __name__ == "__main__": - - def xx(t, x): - exception_err(t, x) - - xx("Exception", "test!!") diff --git a/newapi/api_utils/printe.py b/newapi/api_utils/printe.py index ebf1d75..e0da352 100644 --- a/newapi/api_utils/printe.py +++ b/newapi/api_utils/printe.py @@ -732,16 +732,3 @@ def test_print(s): "info", "test_print", ] - -if __name__ == "__main__": - line = "" - numb = 0 - for co, cac in color_table.items(): - if cac: - numb += 1 - line += f" {co.ljust(15)} <<{co}>> test.<>" - line += "\n" - # if numb % 5 == 0: line += "\n" - # --- - output(line) - showDiff(line, f"{line}3434s") diff --git a/newapi/api_utils/txtlib.py b/newapi/api_utils/txtlib.py index dd6559d..af67568 100644 --- a/newapi/api_utils/txtlib.py +++ b/newapi/api_utils/txtlib.py @@ -104,35 +104,3 @@ def get_all_temps_params(text, templates=None, lowers=False): tab = get_one_temp_params(text, templates=templates, lowers=lowers, get_all_temps=True) # --- return tab - - -# --- -test_text = """ -{{ص.م/صورة مضاعفة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}} -| صورة1 ={{{علم|{{{flag|{{{صورة علم|}}}}}}}}} -| تعليق1 ={{#لو:{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{وصف العلم|{{{flagcaption|}}}}}}|خاصية=P163|rank=best}}|{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{وصف العلم|{{{flagcaption|}}}}}}|خاصية=P163|rank=best}}|{{فصع}}}} -| عرض1 ={{{عرض العلم|{{{flagsize|125}}}}}} -| صورة2 ={{{motto|{{{شعار|}}}}}} -| تعليق2 ={{#لو:{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{تعليق الشعار|{{{وصف الشعار|}}}}}}|خاصية=P237|rank=best}}|{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{تعليق الشعار|{{{وصف الشعار|}}}}}}|خاصية=P237|rank=best}}|{{فصع}}}} -| عرض2 = {{{عرض الشعار|125}}} -| خاصية1 =P41 -| خاصية2 ={{#لو:{{#خاصية:P94}}|P94|P154}} -|خلفية={{{خلفية|}}} -}} - -{{ourworldindatamirror|https://owidm.wmcloud.org/grapher/cancer-death-rates?tab=map {{Webarchive}}}} -""" -# --- -if __name__ == "__main__": - # --- - # --- - ingr = extract_templates_and_params(test_text) - for temp in ingr: - # --- - name, namestrip, params, template = temp["name"], temp["namestrip"], temp["params"], temp["item"] - # --- - print("-----------------------------") - print(f"name: {name}") - print(f"namestrip: {namestrip}") - print(f"params: {params}") - print(f"template: {template}") diff --git a/newapi/pformat.py b/newapi/pformat.py index 5547640..61a124f 100644 --- a/newapi/pformat.py +++ b/newapi/pformat.py @@ -28,39 +28,3 @@ def make_new_text(text): new_text = new_text.replace(temp_text, new_temp) # --- return new_text - - -if __name__ == "__main__": - # --- - sys.argv.append("workibrahem") - sys.argv.append("ask") - # --- - Dir = Path(__file__).parent - # --- - title = "" - text = open(f"{Dir}/pformat.txt", "r", encoding="utf-8").read() - for arg in sys.argv: - arg, _, value = arg.partition(":") - if arg == "-title" or arg == "-page": - title = value - # --- - if title: - # --- - from newapi.page import MainPage - - page = MainPage(title, "ar", family="wikipedia") - text = page.get_text() - newtext = make_new_text(text) - if "save" in sys.argv: - save_page = page.save(newtext=newtext, summary="", nocreate=1, minor="") - else: - prased = wtp.parse(text) - newtext = prased.pformat() - # --- - print(newtext) - # --- - with open(f"{Dir}/pformat.txt", "w", encoding="utf-8") as logfile: - logfile.write(newtext) - # --- - print('add "save" to sys.argv to save.') - # --- diff --git a/newapi/super/S_Page/ar_err.py b/newapi/super/S_Page/ar_err.py index c4f5d82..2c680c3 100644 --- a/newapi/super/S_Page/ar_err.py +++ b/newapi/super/S_Page/ar_err.py @@ -40,8 +40,3 @@ def test_find_edit_error(): print(f"Test case 3: Result = {result}") print("All test cases pass!") - - -if __name__ == "__main__": - # Run the test - test_find_edit_error() From 6f8c7d79ea8c336ddd4efe3986deb0de0f92d855 Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Mon, 2 Feb 2026 04:35:41 +0300 Subject: [PATCH 10/11] Update super_page.py --- newapi/super/S_Page/super_page.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/newapi/super/S_Page/super_page.py b/newapi/super/S_Page/super_page.py index 9904caa..8d7e360 100644 --- a/newapi/super/S_Page/super_page.py +++ b/newapi/super/S_Page/super_page.py @@ -336,15 +336,15 @@ def get_infos(self): if "sortkey" in cat: del cat["sortkey"] # --- - tit = cat["title"] + category_title = cat["title"] # --- - self.categories_data.all_categories_with_hidden[tit] = cat + self.categories_data.all_categories_with_hidden[category_title] = cat # --- if cat.get("hidden") is True: - self.categories_data.hidden_categories[tit] = cat + self.categories_data.hidden_categories[category_title] = cat else: del cat["hidden"] - self.categories_data.categories[tit] = cat + self.categories_data.categories[category_title] = cat # --- if ta.get("langlinks", []) != []: # --- @@ -884,6 +884,9 @@ def Create(self, text="", summary="", nodiff="", noask=False): # --- return False + def create(self, text="", summary="", nodiff="", noask=False): + return self.Create(text=text, summary=summary, nodiff=nodiff, noask=noask) + def page_backlinks(self, ns=0): params = { "action": "query", From 5946368534dbb0ef143bdf5abefdc6a8711a3f6d Mon Sep 17 00:00:00 2001 From: Ibrahem Date: Tue, 17 Feb 2026 02:45:40 +0300 Subject: [PATCH 11/11] . --- newapi/super/Login_db/__init__.py | 1 - newapi/super/Login_db/bot.py | 163 ------------------------------ newapi/super/Login_db/bot1.py | 61 ----------- newapi/super/Login_db/lite.py | 40 -------- newapi/super/bot.py | 6 +- newapi/super/bot_new.py | 8 +- newapi/super/mwclient/client.py | 22 ++-- 7 files changed, 16 insertions(+), 285 deletions(-) delete mode 100644 newapi/super/Login_db/__init__.py delete mode 100644 newapi/super/Login_db/bot.py delete mode 100644 newapi/super/Login_db/bot1.py delete mode 100644 newapi/super/Login_db/lite.py diff --git a/newapi/super/Login_db/__init__.py b/newapi/super/Login_db/__init__.py deleted file mode 100644 index 40a96af..0000000 --- a/newapi/super/Login_db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/newapi/super/Login_db/bot.py b/newapi/super/Login_db/bot.py deleted file mode 100644 index 83af18f..0000000 --- a/newapi/super/Login_db/bot.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -Module for handling login attempt logging to database - -from newapi.super.Login_db.bot2 import log_one - -# log_one(site="", user="", result="", db_type="sqlite", db_path="login_logs.db", credentials=None) - -""" - -import datetime -import functools -import os -import sys -from configparser import ConfigParser - -from ...api_utils import printe -from ...DB_bots.pymysql_bot import sql_connect_pymysql - -LOGS_IS_ENABLED = os.getenv("LOGIN_LOGS_IS_ENABLED", "0") == "1" - - -def local_host(): - main_args = { - "host": "127.0.0.1", - "db": "mv", - "charset": "utf8mb4", - # "cursorclass": Typee, - "use_unicode": True, - "autocommit": True, - } - - credentials = {"user": "root", "password": "root11"} - - return main_args, credentials - - -@functools.lru_cache(maxsize=1) -def jls_extract_def(): - # --- - if not os.getenv("HOME"): - return local_host() - # --- - project = os.getenv("HOME") - # --- - if __file__.find("/data/project/himo") != -1: - project = "/data/project/himo" - # --- - db_connect_file = f"{project}/confs/db.ini" - db_username = "" - credentials = {"read_default_file": db_connect_file} - # --- - # if db_connect_file is not file - if os.path.isfile(db_connect_file): - # --- - try: - config2 = ConfigParser() - config2.read(db_connect_file) - db_username = config2["client"]["user"] - except KeyError as e: - print(f"error: {e}") - # --- - # Typee = pymysql.cursors.DictCursor if return_dict else pymysql.cursors.Cursor - # --- - main_args = { - "host": "tools.db.svc.wikimedia.cloud", - "db": f"{db_username}__mv", - "charset": "utf8mb4", - # "cursorclass": Typee, - "use_unicode": True, - "autocommit": True, - } - - if os.getenv("HOME").find("/data/project/mdwiki") != -1: - main_args["db"] = f"{db_username}__mdwiki_new" - - return main_args, credentials - - -main_args, credentials = jls_extract_def() - -userinfo = 0 - - -def log_one(site, user, result, action="", params={}) -> None: - # global userinfo - # --- - if not LOGS_IS_ENABLED: - return None - # --- - params = params or {} - # --- - if action == "query": - if params.get("meta", "") == "tokens": - action = "tokens" - if params.get("type"): - action += "_" + params["type"] - elif params.get("meta", "").find("userinfo") != -1: - action = "userinfo" - # --- - # if params.get('meta', "").find("userinfo") != -1: - # userinfo += 1 - # if userinfo > 3: - # raise Exception("too much") - # --- - if action == "query" and params.get("prop"): - action += "_" + "_".join(params["prop"].split("|")) - # --- - if action == "query" and "test" in sys.argv: - printe.output(f"<> {action=}") - print(dict(params)) - # --- - # Create table query - _create_table_query = """ - CREATE TABLE IF NOT EXISTS login_attempts ( - id INT AUTO_INCREMENT PRIMARY KEY, - site VARCHAR(255) NOT NULL, - username VARCHAR(255) NOT NULL, - result VARCHAR(50) NOT NULL, - timestamp DATETIME NOT NULL, - action VARCHAR(100) NULL, - INDEX idx_site_user (site, username), - INDEX idx_timestamp (timestamp) - ) - """ - # Create table query - create_table_query = """ - CREATE TABLE IF NOT EXISTS `logins` ( - `id` int NOT NULL AUTO_INCREMENT, - `site` varchar(255) NOT NULL, - `username` varchar(255) NOT NULL, - `result` varchar(50) NOT NULL, - `action` varchar(100) DEFAULT NULL, - `count` int NOT NULL DEFAULT '1', - `first` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `last` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `site_username_result_action` (`site`,`username`,`result`,`action`), - KEY `idx_site_user` (`site`,`username`), - KEY `idx_timestamp` (`first`) - ) - """ - # print(f"create_table_query: {create_table_query}") - - timestamp = datetime.datetime.now().isoformat() - - # Execute table creation - sql_connect_pymysql(create_table_query, main_args=main_args, credentials=credentials) - - # Insert login attempt - insert_query = """ - INSERT INTO logins (site, username, result, action, count, first, last) - VALUES (%s, %s, %s, %s, %s, NOW(), NOW()) - ON DUPLICATE KEY UPDATE count = count + 1, last = NOW() - """ - - # print(f"insert_query: {insert_query}") - - sql_connect_pymysql( - insert_query, - values=(site, user, result, action, 1), - main_args=main_args, - credentials=credentials, - ) diff --git a/newapi/super/Login_db/bot1.py b/newapi/super/Login_db/bot1.py deleted file mode 100644 index e1cb9e3..0000000 --- a/newapi/super/Login_db/bot1.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Module for handling login attempt logging to database - -from newapi.super.Login_db.bot import log_one - -# log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=login_result) - -""" - -import datetime -import json -import os -import sys - -import pymysql - -DB_CONFIG = { - "host": "localhost", - "user": "root", - "password": "password", - "database": "login_db", - "charset": "utf8mb4", - "cursorclass": pymysql.cursors.DictCursor, -} - -LOGS_IS_ENABLED = os.getenv("LOGIN_LOGS_IS_ENABLED", "0") == "1" - - -def log_one(site="", user="", result=""): - """ - Logs a login attempt to the database - - Args: - site (str): The site where the login attempt occurred - user (str): The username used in the attempt - result (str): The result of the attempt (success/failure) - """ - # --- - if not LOGS_IS_ENABLED: - return - # --- - timestamp = datetime.datetime.now().isoformat() - # --- - insert_query = """ - INSERT INTO login_attempts (site, username, result, timestamp) - VALUES (%s, %s, %s, %s) - """ - # --- - try: - connection = pymysql.connect(**DB_CONFIG) - try: - with connection.cursor() as cursor: - cursor.execute(insert_query, (site, user, result, timestamp)) - connection.commit() - except Exception as e: - connection.rollback() - print(f"Database error occurred: {e}") - finally: - connection.close() - except pymysql.Error as e: - print(f"Database error occurred: {e}") diff --git a/newapi/super/Login_db/lite.py b/newapi/super/Login_db/lite.py deleted file mode 100644 index c99bd15..0000000 --- a/newapi/super/Login_db/lite.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Module for handling login attempt logging to database - -from newapi.super.Login_db.bot2 import log_one - -# log_one(site="", user="", result="", db_type="sqlite", db_path="login_logs.db", credentials=None) - -""" - -import datetime -from pathlib import Path - -from ...api_utils.except_err import exception_err -from ...DB_bots import db_bot - -LiteDB = db_bot.LiteDB - - -def log_one_sqlite(site="", user="", result=""): - Dir = Path(__file__).resolve().parent - - db_path = Dir / "login_logs.db" - - timestamp = datetime.datetime.now().isoformat() - - try: - db = LiteDB(db_path) - - # Create table if it doesn't exist - db.create_table( - "login_attempts", {"id": int, "site": str, "username": str, "result": str, "timestamp": str}, pk="id" - ) - - # Insert login attempt - db.insert( - "login_attempts", {"site": site, "username": user, "result": result, "timestamp": timestamp}, check=False - ) # Don't check for duplicates since each attempt should be logged - - except Exception as e: - exception_err(e, "Failed to log login attempt to SQLite") diff --git a/newapi/super/bot.py b/newapi/super/bot.py index d4b094e..fc3b8dd 100644 --- a/newapi/super/bot.py +++ b/newapi/super/bot.py @@ -8,6 +8,7 @@ """ import os +import logging import sys from http.cookiejar import MozillaCookieJar @@ -17,14 +18,13 @@ from ..api_utils.except_err import exception_err from ..api_utils.user_agent import default_user_agent from .cookies_bot import del_cookies_file, get_file_name -from .Login_db.bot import log_one from .params_help import PARAMS_HELPS # cookies = get_cookies(lang, family, username) seasons_by_lang = {} users_by_lang = {} logins_count = {1: 0} - +logger = logging.getLogger(__name__) botname = "newapi" @@ -58,7 +58,7 @@ def __init__(self) -> None: super().__init__() def log_error(self, result, action, params=None) -> None: - log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=result, action=action, params=params) + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def add_User_tables(self, family, table, lang="") -> None: # --- diff --git a/newapi/super/bot_new.py b/newapi/super/bot_new.py index 006bbd5..94d7034 100644 --- a/newapi/super/bot_new.py +++ b/newapi/super/bot_new.py @@ -7,6 +7,7 @@ """ import copy +import logging import os import sys from http.cookiejar import MozillaCookieJar @@ -17,13 +18,10 @@ from ..api_utils.except_err import exception_err from ..api_utils.user_agent import default_user_agent from .cookies_bot import del_cookies_file, get_file_name -from .Login_db.bot import log_one - -# from mwclient.client import Site from .mwclient.client import Site from .params_help import PARAMS_HELPS -# import mwclient +logger = logging.getLogger(__name__) # cookies = get_cookies(lang, family, username) @@ -49,7 +47,7 @@ def __init__(self, lang, family): # self._start_() def log_error(self, result, action, params=None) -> None: - log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=result, action=action, params=params) + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def _start_(self, username, password): self.username = username diff --git a/newapi/super/mwclient/client.py b/newapi/super/mwclient/client.py index b9935d9..9d0db6e 100644 --- a/newapi/super/mwclient/client.py +++ b/newapi/super/mwclient/client.py @@ -6,8 +6,6 @@ import requests from requests.auth import AuthBase, HTTPBasicAuth from requests_oauthlib import OAuth1 - -from ..Login_db.bot import log_one from . import errors, listing # from .sleep import Sleepers @@ -15,7 +13,7 @@ __version__ = "0.11.0" -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) USER_AGENT = "mwclient/{} ({})".format(__version__, "https://github.com/mwclient/mwclient") @@ -391,7 +389,7 @@ def api(self, action, http_method="POST", *args, **kwargs): return info def log_error(self, result, action, params=None) -> None: - log_one(site=self.host, user=self.username, result=result, action=action, params=params) + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def handle_api_result(self, info, kwargs=None, sleeper=None): """Checks the given API response, raising an appropriate exception or sleeping if @@ -423,7 +421,7 @@ def handle_api_result(self, info, kwargs=None, sleeper=None): if "warnings" in info: for module, warning in info["warnings"].items(): if "*" in warning: - log.warning(warning["*"]) + logger.warning(warning["*"]) if "error" in info: if info["error"].get("code") in {"internal_api_error_DBConnectionError", "internal_api_error_DBQueryError"}: @@ -434,7 +432,7 @@ def handle_api_result(self, info, kwargs=None, sleeper=None): if info["error"].get("code") == "mwoauth-invalid-authorization" and "Nonce already used" in info[ "error" ].get("info"): - log.warning("Retrying due to nonce error, see" "https://phabricator.wikimedia.org/T106066") + logger.warning("Retrying due to nonce error, see" "https://phabricator.wikimedia.org/T106066") # sleeper.sleep() return False @@ -519,7 +517,7 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method="P stream = self.connection.request(http_method, url, **args) if stream.headers.get("x-database-lag"): wait_time = int(stream.headers.get("retry-after")) - log.warning("Database lag exceeds max lag. " f"Waiting for {wait_time} seconds, maxlag:{maxlag}") + logger.warning("Database lag exceeds max lag. " f"Waiting for {wait_time} seconds, maxlag:{maxlag}") # fall through to the sleep elif stream.status_code == 200: return stream.text @@ -528,7 +526,7 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method="P else: if not retry_on_error: stream.raise_for_status() - log.warning( + logger.warning( "Received {status} response: {text}. " "Retrying in a moment.".format(status=stream.status_code, text=stream.text) ) @@ -543,7 +541,7 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method="P if not retry_on_error: print("raise") print(err) - log.warning("Connection error. Retrying in a moment.") + logger.warning("Connection error. Retrying in a moment.") toraise = err # proceed to the sleep @@ -771,7 +769,7 @@ def login(self, username=None, password=None, cookies=None, domain=None): try: kwargs["lgtoken"] = self.get_token("login") except (errors.APIError, KeyError): - log.debug("Failed to get login token, MediaWiki is older than 1.27.") + logger.debug("Failed to get login token, MediaWiki is older than 1.27.") # while True: login = self.post("login", **kwargs) @@ -847,7 +845,7 @@ def clientlogin(self, cookies=None, **kwargs): try: kwargs["logintoken"] = self.get_token("login") except (errors.APIError, KeyError): - log.debug("Failed to get login token, MediaWiki is older than 1.27.") + logger.debug("Failed to get login token, MediaWiki is older than 1.27.") if "logincontinue" not in kwargs and "loginreturnurl" not in kwargs: # should be great if API didn't require this... @@ -1090,7 +1088,7 @@ def chunk_upload(self, file, filename, ignorewarnings, comment, text): offset += chunk.tell() chunk.close() - log.debug("%s: Uploaded %d of %d bytes", filename, offset, content_size) + logger.debug("%s: Uploaded %d of %d bytes", filename, offset, content_size) params["filekey"] = response["filekey"] if response["result"] == "Continue": params["offset"] = response["offset"]