diff --git a/Diagram.md b/Diagram.md index 200dfae..d25345b 100644 --- a/Diagram.md +++ b/Diagram.md @@ -34,7 +34,6 @@ flowchart TD click PageOps "https://github.com/wikiar/newapi/blob/main/page.py" click PageOps "https://github.com/wikiar/newapi/blob/main/wiki_page.py" click PageOps "https://github.com/wikiar/newapi/blob/main/mdwiki_page.py" - click PageOps "https://github.com/wikiar/newapi/blob/main/ncc_page.py" click PageOps "https://github.com/wikiar/newapi/tree/main/super/page_bots/" %% Click Events for Bot Functionality Modules diff --git a/README1.md b/README1.md deleted file mode 100644 index 1ac0e14..0000000 --- a/README1.md +++ /dev/null @@ -1,113 +0,0 @@ -# Wikimedia API Python Module - -## Overview -This project is a Python module designed to interact with the Wikimedia API. It provides a unified, programmable interface for common Wikimedia actions such as: -- Logging in -- Retrieving and editing pages -- Working with categories and templates -- Performing various Wikimedia API actions - -The module abstracts the Wikimedia API into user-friendly functions and classes, reducing the need for direct HTTP request handling and token management. - -## Features -- **API Interface Layer**: Encapsulates core Wikimedia interactions, handling requests and responses. -- **Login Module**: Simplifies authentication with multiple bot-based login implementations. -- **Page Management Module**: Offers functionality to check page existence, edit pages, and manage page elements. -- **Category and Template Handling**: Supports category depth processing and template handling. -- **Bot Modules**: Automates API interactions and page operations. -- **MWClient Wrappers**: Provides lower-level API communication support. -- **Database & Utility Bots**: Manages database interactions and caching. -- **Testing & CI/CD Support**: Includes a test suite and GitHub workflows for automated validation. - -## System Architecture -The project follows a modular and layered architecture: -1. **Client/Consumer Applications** interact with the module. -2. **API Interface Layer** (e.g., `NEW_API` class) abstracts direct Wikimedia API calls. -3. **Login Module** handles session authentication. -4. **Page Management Module** enables page retrieval, editing, and processing. -5. **Category and Template Handling** supports Wikimedia-specific structures. -6. **Bot Modules** automate repetitive Wikimedia tasks. -7. **MWClient Wrappers** facilitate low-level API interactions. -8. **Database & Utility Bots** provide storage and session support. -9. **Testing & CI/CD** ensures reliability through automated testing and configuration management. - -## File Mapping -### API Interface Layer -- `wiki_page.py`, `page.py`, `mdwiki_page.py`, `ncc_page.py` -- `super/super_page.py` - -### Login Module -- `super/super_login.py` -- `super/S_Login/` (including `bot.py`, `bot_new.py`, `cookies_bot.py`) - -### Page Management Module -- `wiki_page.py`, `page.py`, `mdwiki_page.py`, `ncc_page.py` -- `super/super_page.py` - -### Category and Template Handling -- `super/catdepth_new.py` - -### Bot Modules -- API bots: `super/botapi_bots/` -- Page bots: `super/page_bots/` -- Additional bot utilities: `botEdit.py`, `super/bots/` - -### MWClient Wrappers -- `super/S_Login/mwclient/` (including `client.py`, `page.py`, `errors.py`, `image.py`, `listing.py`, `sleep.py`, `util.py`) - -### Database & Utility Bots -- `db_bot.py`, `pymysql_bot.py` - -### Testing Suite -- `tests/` (e.g., `tests/test_bot_api.py`, `tests/test_db_bot.py`) - -### CI/CD & Configuration -- `.github/` (GitHub workflows and issue templates) -- YAML configuration: `sweep.yaml`, `.coderabbit.yaml` -- Dependencies: `requirements.in` - -## System Design Diagram Guidelines -To accurately visualize the architecture: -1. **Identify Main Components**: Include API layers, login, page management, bots, and utility modules. -2. **Map Relationships**: - - Client → Login Module → MWClient Wrapper → API Interface → Page Management - - Bots interact with API Interface and Page Management -3. **Use Layered Representation**: - - High-level API components (NEW_API, MainPage) - - Mid-level modules (bot handlers, database utilities) - - Low-level API wrappers (MWClient interactions) -4. **Diagram Elements**: - - Labeled boxes for components - - Directional arrows to indicate data flow - - Different colors for API layers, bots, and utility modules - - Sub-diagrams for clusters like the `super` directory - -## Installation & Usage -### Installation -```sh -pip install -r requirements.in -``` - -### Usage Example -```python -from newapi.page import MainPage - -# Initialize API -page = MainPage("Example_Page") - -# Check if the page exists -if page.exists(): - print("Page found!") - -# Edit a page -page.edit("Updated content") -``` - -## Contribution -1. Fork the repository -2. Create a new branch -3. Make changes and commit -4. Submit a pull request - -## License -This project is licensed under the MIT License. diff --git a/newapi/DB_bots/db_bot.py b/newapi/DB_bots/db_bot.py index 41330fa..9e15f11 100644 --- a/newapi/DB_bots/db_bot.py +++ b/newapi/DB_bots/db_bot.py @@ -10,6 +10,7 @@ # db.select(table_name, args) """ + # --- import sqlite_utils diff --git a/newapi/DB_bots/pymysql_bot.py b/newapi/DB_bots/pymysql_bot.py index bb6d43c..7be0cc5 100644 --- a/newapi/DB_bots/pymysql_bot.py +++ b/newapi/DB_bots/pymysql_bot.py @@ -2,13 +2,18 @@ from newapi import pymysql_bot # result = pymysql_bot.sql_connect_pymysql(query, return_dict=False, values=None, main_args={}, credentials={}, conversions=None) """ + import copy + import pymysql import pymysql.cursors + from ..api_utils.except_err import exception_err -def sql_connect_pymysql(query, return_dict=False, values=None, main_args={}, credentials={}, conversions=None, many=False, **kwargs): +def sql_connect_pymysql( + query, return_dict=False, values=None, main_args={}, credentials={}, conversions=None, many=False, **kwargs +): args = copy.deepcopy(main_args) args["cursorclass"] = pymysql.cursors.DictCursor if return_dict else pymysql.cursors.Cursor if conversions: diff --git a/newapi/__init__.py b/newapi/__init__.py index e29eeef..48d76c8 100644 --- a/newapi/__init__.py +++ b/newapi/__init__.py @@ -2,13 +2,12 @@ """ from newapi import useraccount """ -from .DB_bots import db_bot, pymysql_bot -from .accounts import useraccount -from .api_utils import except_err, botEdit -from .api_utils import printe, txtlib, wd_sparql -from .super.login_wrap import LoginWrap from . import page +from .accounts import useraccount from .all_apis import ALL_APIS +from .api_utils import botEdit, except_err, printe, txtlib, wd_sparql +from .DB_bots import db_bot, pymysql_bot +from .super.login_wrap import LoginWrap __all__ = [ "ALL_APIS", diff --git a/newapi/accounts/user_account_ncc.py b/newapi/accounts/user_account_ncc.py deleted file mode 100644 index 7a63f4e..0000000 --- a/newapi/accounts/user_account_ncc.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -# --- -from .accounts import user_account_ncc -# --- -User_tables = { - "username": user_account_ncc.username, - "password": user_account_ncc.password, -} -# --- -""" - -import os -import configparser - -home_dir = os.getenv("HOME") -project = home_dir if home_dir else "I:/ncc" -# --- -config = configparser.ConfigParser() -config.read(f"{project}/confs/nccommons_user.ini") -# --- -username = config["DEFAULT"].get("username", "").strip() -password = config["DEFAULT"].get("password", "").strip() - -User_tables = { - "username": username, - "password": password, -} - -SITECODE = "www" -FAMILY = "nccommons" diff --git a/newapi/accounts/user_account_new.py b/newapi/accounts/user_account_new.py deleted file mode 100644 index 999068f..0000000 --- a/newapi/accounts/user_account_new.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -# --- -from . import user_account_new -# --- -username = user_account_new.bot_username #user_account_new.my_username -password = user_account_new.bot_password #user_account_new.mdwiki_pass -lgpass_enwiki = user_account_new.lgpass_enwiki -user_agent = user_account_new.user_agent -# --- -""" - -import os -import configparser - -home_dir = os.getenv("HOME") -project = home_dir if home_dir else "I:/mdwiki/mdwiki" -# --- -config = configparser.ConfigParser() -config.read(f"{project}/confs/user.ini") - -username = config["DEFAULT"].get("botusername", "") -password = config["DEFAULT"].get("botpassword", "") - -bot_username = config["DEFAULT"].get("botusername", "") -bot_password = config["DEFAULT"].get("botpassword", "") - -my_username = config["DEFAULT"].get("my_username", "") - -mdwiki_pass = config["DEFAULT"].get("mdwiki_pass", "") - -lgpass_enwiki = config["DEFAULT"].get("lgpass_enwiki", "") -my_password = lgpass_enwiki - -qs_token = config["DEFAULT"].get("qs_token", "") - -user_agent = config["DEFAULT"].get("user_agent", "") - -User_tables = { - "username": my_username, - "password": mdwiki_pass, -} - -User_tables_wiki = { - "username": bot_username, - "password": bot_password, -} - -SITECODE = "www" -FAMILY = "mdwiki" diff --git a/newapi/accounts/useraccount.py b/newapi/accounts/useraccount.py index a9c9584..b8ec71c 100644 --- a/newapi/accounts/useraccount.py +++ b/newapi/accounts/useraccount.py @@ -3,39 +3,40 @@ from newapi.accounts.useraccount import User_tables_bot, User_tables_ibrahem """ -import sys -import os + import configparser +import os +import sys -project = '/data/project/himo' +project = "/data/project/himo" # --- if not os.path.isdir(project): - project = 'I:/core/bots/core1' + project = "I:/core/bots/core1" # --- config = configparser.ConfigParser() config.read(f"{project}/confs/user.ini") -DEFAULT = config['DEFAULT'] +DEFAULT = config["DEFAULT"] -username = config['DEFAULT'].get('botusername', "") -password = config['DEFAULT'].get('botpassword', "") +username = config["DEFAULT"].get("botusername", "") +password = config["DEFAULT"].get("botpassword", "") # --- -passworden = config['DEFAULT'].get('passworden', "") -passwordwd = config['DEFAULT'].get('passwordwd', "") +passworden = config["DEFAULT"].get("passworden", "") +passwordwd = config["DEFAULT"].get("passwordwd", "") -password_ar = config['DEFAULT'].get('password_ar', "") -password_en = config['DEFAULT'].get('password_en', "") +password_ar = config["DEFAULT"].get("password_ar", "") +password_en = config["DEFAULT"].get("password_en", "") -hiacc = config['DEFAULT'].get('hiacc', "") -hipass = config['DEFAULT'].get('hipass', "") +hiacc = config["DEFAULT"].get("hiacc", "") +hipass = config["DEFAULT"].get("hipass", "") -mdwiki_pass = config['DEFAULT'].get('mdwiki_pass', "") +mdwiki_pass = config["DEFAULT"].get("mdwiki_pass", "") -qs_token = config['DEFAULT'].get('qs_token', "") -qs_tokenbot = config['DEFAULT'].get('qs_tokenbot', "") +qs_token = config["DEFAULT"].get("qs_token", "") +qs_tokenbot = config["DEFAULT"].get("qs_tokenbot", "") -user_agent = config['DEFAULT'].get('user_agent', "") +user_agent = config["DEFAULT"].get("user_agent", "") if "workibrahem" in sys.argv: username = hiacc diff --git a/newapi/all_apis.py b/newapi/all_apis.py index e55c0c5..e7e000c 100644 --- a/newapi/all_apis.py +++ b/newapi/all_apis.py @@ -1,5 +1,5 @@ -""" -""" +""" """ + from .pages_bots.all_apis import ( ALL_APIS, ) diff --git a/newapi/api_utils/ask_bot.py b/newapi/api_utils/ask_bot.py index 685c49f..3a99894 100644 --- a/newapi/api_utils/ask_bot.py +++ b/newapi/api_utils/ask_bot.py @@ -3,13 +3,16 @@ from ...api_utils.ask_bot import ASK_BOT """ + import sys + from . import printe yes_answer = ["y", "a", "", "Y", "A", "all", "aaa"] Save_or_Ask = {} + class ASK_BOT: def __init__(self): pass diff --git a/newapi/api_utils/botEdit.py b/newapi/api_utils/botEdit.py index 86806ba..1a9b768 100644 --- a/newapi/api_utils/botEdit.py +++ b/newapi/api_utils/botEdit.py @@ -2,10 +2,12 @@ from newapi.api_utils import botEdit bot_edit! """ + +from .bot_edit.bot_edit_by_templates import is_bot_edit_allowed + # # from .bot_edit.bot_edit_by_time import check_create_time, check_last_edit_time -from .bot_edit.bot_edit_by_templates import is_bot_edit_allowed Created_Cache = {} diff --git a/newapi/api_utils/bot_edit/bot_edit_by_templates.py b/newapi/api_utils/bot_edit/bot_edit_by_templates.py index 2aa7f8b..0abb584 100644 --- a/newapi/api_utils/bot_edit/bot_edit_by_templates.py +++ b/newapi/api_utils/bot_edit/bot_edit_by_templates.py @@ -2,10 +2,12 @@ from newapi.api_utils import botEdit bot_edit! """ + # # import logging import sys + import wikitextparser as wtp logger = logging.getLogger(__name__) @@ -126,7 +128,7 @@ def is_bot_edit_allowed(text="", title_page="", botjob="all"): title = str(template.normal_name()).strip() # --- params = { - str(param.name).strip() : str(param.value).strip() + str(param.name).strip(): str(param.value).strip() for param in template.arguments if str(param.value).strip() } @@ -140,7 +142,7 @@ def is_bot_edit_allowed(text="", title_page="", botjob="all"): Bot_Cache[botjob][title_page] = False return False # --- - logger.debug('<>botEdit.py title:(%s), params:(%s).' % (title, str(params))) + logger.debug("<>botEdit.py title:(%s), params:(%s)." % (title, str(params))) # --- if title.lower() == "nobots": return _handle_nobots_template(params, title_page, botjob, _template) diff --git a/newapi/api_utils/bot_edit/bot_edit_by_time.py b/newapi/api_utils/bot_edit/bot_edit_by_time.py index 85578c5..6bce11a 100644 --- a/newapi/api_utils/bot_edit/bot_edit_by_time.py +++ b/newapi/api_utils/bot_edit/bot_edit_by_time.py @@ -1,7 +1,9 @@ -""" -""" +""" """ + import datetime + from .. import printe + Created_Cache = {} @@ -80,7 +82,9 @@ def check_last_edit_time(page, title_page, delay): # --- if diff_minutes < delay: printe.output(f"<>Page:{title_page} last edit ({timestamp}).") - printe.output(f"<>Page Last edit before {delay} minutes, Wait {wait_time:.2f} minutes. title:{title_page}") + printe.output( + f"<>Page Last edit before {delay} minutes, Wait {wait_time:.2f} minutes. title:{title_page}" + ) return False # --- return True diff --git a/newapi/api_utils/except_err.py b/newapi/api_utils/except_err.py index 98c26aa..ecbf658 100644 --- a/newapi/api_utils/except_err.py +++ b/newapi/api_utils/except_err.py @@ -6,6 +6,7 @@ from .except_err import exception_err, warn_err """ + import inspect import traceback from warnings import warn @@ -49,11 +50,3 @@ def exception_err(e, text=""): # --- printe.warn("CRITICAL:") # printe.info("====") - - -if __name__ == "__main__": - - def xx(t, x): - exception_err(t, x) - - xx("Exception", "test!!") diff --git a/newapi/api_utils/printe.py b/newapi/api_utils/printe.py index 8be1b76..e0da352 100644 --- a/newapi/api_utils/printe.py +++ b/newapi/api_utils/printe.py @@ -9,18 +9,17 @@ printe.showDiff('old text', 'new text') # prints the differences between 'old text' and 'new text' """ +import difflib + # --- import functools -import difflib +import logging import re import sys - from collections import abc +from collections.abc import Iterable, Sequence from difflib import _format_range_unified as format_range_unified from itertools import zip_longest -from collections.abc import Iterable, Sequence - -import logging if "debug" in sys.argv: logging.basicConfig(level=logging.DEBUG) @@ -230,7 +229,7 @@ def get_color_table(): "lightblack": 108, "bold": 1, } - color_table = {x : f"\033[{v}m%s\033[00m" for x, v in color_numbers.items()} + color_table = {x: f"\033[{v}m%s\033[00m" for x, v in color_numbers.items()} # Add light versions of the colors to the color table for color in ["purple", "yellow", "blue", "red", "green", "cyan", "gray"]: @@ -344,7 +343,9 @@ class Hunk: NOT_APPR = -1 PENDING = 0 - def __init__(self, a: str | Sequence[str], b: str | Sequence[str], grouped_opcode: Sequence[tuple[str, int, int, int, int]]) -> None: + def __init__( + self, a: str | Sequence[str], b: str | Sequence[str], grouped_opcode: Sequence[tuple[str, int, int, int, int]] + ) -> None: """ Initializer. @@ -537,7 +538,9 @@ def get_header_text(a_rng: tuple[int, int], b_rng: tuple[int, int], affix: str = class PatchManager: - def __init__(self, text_a: str, text_b: str, context: int = 0, by_letter: bool = False, replace_invisible: bool = False) -> None: + def __init__( + self, text_a: str, text_b: str, context: int = 0, by_letter: bool = False, replace_invisible: bool = False + ) -> None: self.a = text_a.splitlines(True) self.b = text_b.splitlines(True) @@ -629,7 +632,10 @@ def _get_context_range(self, super_hunk: _Superhunk) -> tuple[tuple[int, int], t """Dynamically determine context range for a super hunk.""" a0, a1 = super_hunk.a_rng b0, b1 = super_hunk.b_rng - return ((a0 - min(super_hunk.pre_context, self.context), a1 + min(super_hunk.post_context, self.context)), (b0 - min(super_hunk.pre_context, self.context), b1 + min(super_hunk.post_context, self.context))) + return ( + (a0 - min(super_hunk.pre_context, self.context), a1 + min(super_hunk.post_context, self.context)), + (b0 - min(super_hunk.pre_context, self.context), b1 + min(super_hunk.post_context, self.context)), + ) def _generate_diff(self, hunks: _Superhunk) -> str: """Generate a diff text for the given hunks.""" @@ -726,16 +732,3 @@ def test_print(s): "info", "test_print", ] - -if __name__ == "__main__": - line = "" - numb = 0 - for co, cac in color_table.items(): - if cac: - numb += 1 - line += f" {co.ljust(15)} <<{co}>> test.<>" - line += "\n" - # if numb % 5 == 0: line += "\n" - # --- - output(line) - showDiff(line, f"{line}3434s") diff --git a/newapi/api_utils/txtlib.py b/newapi/api_utils/txtlib.py index 802c8bb..af67568 100644 --- a/newapi/api_utils/txtlib.py +++ b/newapi/api_utils/txtlib.py @@ -13,9 +13,10 @@ """ # from ..other_bots import printe -import wikitextparser as wtp from functools import lru_cache +import wikitextparser as wtp + @lru_cache(maxsize=512) def extract_templates_and_params(text): @@ -24,7 +25,7 @@ def extract_templates_and_params(text): # --- parsed = wtp.parse(text) templates = parsed.templates - arguments = 'arguments' + arguments = "arguments" # --- for template in templates: # --- @@ -46,10 +47,10 @@ def extract_templates_and_params(text): namestrip = name # --- ficrt = { - 'name': f"قالب:{name}", - 'namestrip': namestrip, - 'params': params, - 'item': pa_item, + "name": f"قالب:{name}", + "namestrip": namestrip, + "params": params, + "item": pa_item, } # --- result.append(ficrt) @@ -65,7 +66,7 @@ def get_one_temp_params(text, tempname="", templates=[], lowers=False, get_all_t if tempname: temps.append(tempname) # --- - temps = [x.replace("قالب:", "").replace("Template:", "").replace('_', ' ').strip() for x in temps] + temps = [x.replace("قالب:", "").replace("Template:", "").replace("_", " ").strip() for x in temps] # --- if lowers: temps = [x.lower() for x in temps] @@ -78,7 +79,7 @@ def get_one_temp_params(text, tempname="", templates=[], lowers=False, get_all_t for temp in ingr: # --- # name, namestrip, params, template = temp['name'], temp['namestrip'], temp['params'], temp['item'] - namestrip, params = temp['namestrip'], temp['params'] + namestrip, params = temp["namestrip"], temp["params"] # --- if lowers: namestrip = namestrip.lower() @@ -89,9 +90,7 @@ def get_one_temp_params(text, tempname="", templates=[], lowers=False, get_all_t # --- # print("te:%s, namestrip:%s" % (te,namestrip) ) # --- - tabe = { - namestrip: params - } + tabe = {namestrip: params} named.append(tabe) # --- return named @@ -105,35 +104,3 @@ def get_all_temps_params(text, templates=None, lowers=False): tab = get_one_temp_params(text, templates=templates, lowers=lowers, get_all_temps=True) # --- return tab - - -# --- -test_text = ''' -{{ص.م/صورة مضاعفة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}} -| صورة1 ={{{علم|{{{flag|{{{صورة علم|}}}}}}}}} -| تعليق1 ={{#لو:{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{وصف العلم|{{{flagcaption|}}}}}}|خاصية=P163|rank=best}}|{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{وصف العلم|{{{flagcaption|}}}}}}|خاصية=P163|rank=best}}|{{فصع}}}} -| عرض1 ={{{عرض العلم|{{{flagsize|125}}}}}} -| صورة2 ={{{motto|{{{شعار|}}}}}} -| تعليق2 ={{#لو:{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{تعليق الشعار|{{{وصف الشعار|}}}}}}|خاصية=P237|rank=best}}|{{قيمة ويكي بيانات|معرف ويكي بيانات={{{معرف ويكي بيانات|}}}|{{{تعليق الشعار|{{{وصف الشعار|}}}}}}|خاصية=P237|rank=best}}|{{فصع}}}} -| عرض2 = {{{عرض الشعار|125}}} -| خاصية1 =P41 -| خاصية2 ={{#لو:{{#خاصية:P94}}|P94|P154}} -|خلفية={{{خلفية|}}} -}} - -{{ourworldindatamirror|https://owidm.wmcloud.org/grapher/cancer-death-rates?tab=map {{Webarchive}}}} -''' -# --- -if __name__ == '__main__': - # --- - # --- - ingr = extract_templates_and_params(test_text) - for temp in ingr: - # --- - name, namestrip, params, template = temp['name'], temp['namestrip'], temp['params'], temp['item'] - # --- - print("-----------------------------") - print(f"name: {name}") - print(f"namestrip: {namestrip}") - print(f"params: {params}") - print(f"template: {template}") diff --git a/newapi/api_utils/user_agent.py b/newapi/api_utils/user_agent.py index c4c83a1..62b9a0b 100644 --- a/newapi/api_utils/user_agent.py +++ b/newapi/api_utils/user_agent.py @@ -3,6 +3,7 @@ from .api_utils.user_agent import default_user_agent """ + import os from functools import lru_cache @@ -18,4 +19,3 @@ def default_user_agent(): # printe.output(f"default_user_agent: {li}") # --- return li - diff --git a/newapi/api_utils/wd_sparql.py b/newapi/api_utils/wd_sparql.py index 61ae757..f46c452 100644 --- a/newapi/api_utils/wd_sparql.py +++ b/newapi/api_utils/wd_sparql.py @@ -6,8 +6,10 @@ get_query_data = wd_sparql.get_query_data """ + import sys -from SPARQLWrapper import SPARQLWrapper, JSON + +from SPARQLWrapper import JSON, SPARQLWrapper def get_query_data(query): diff --git a/newapi/except_err.py b/newapi/except_err.py index 8b1046f..a812440 100644 --- a/newapi/except_err.py +++ b/newapi/except_err.py @@ -1,6 +1,5 @@ -""" +""" """ -""" from .api_utils.except_err import exception_err, warn_err __all__ = [ diff --git a/newapi/mdwiki_page.py b/newapi/mdwiki_page.py deleted file mode 100644 index e79eb83..0000000 --- a/newapi/mdwiki_page.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -""" -from .pages_bots.mdwiki_page import ( - home_dir, - MainPage, - md_MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes -) - -__all__ = [ - "home_dir", - "user_agent", - "MainPage", - "md_MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/newapi/ncc_page.py b/newapi/ncc_page.py deleted file mode 100644 index 0677482..0000000 --- a/newapi/ncc_page.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -""" -from .pages_bots.ncc_page import ( - home_dir, - ncc_MainPage, - MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes -) - -__all__ = [ - "home_dir", - "user_agent", - "ncc_MainPage", - "MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/newapi/new_wiki_pages.py b/newapi/new_wiki_pages.py deleted file mode 100644 index ab04eff..0000000 --- a/newapi/new_wiki_pages.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -""" -from .pages_bots.new_wiki_pages import ( - home_dir, - MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes, -) - -__all__ = [ - "home_dir", - "user_agent", - "MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/newapi/page.py b/newapi/page.py index 026a690..51d7cad 100644 --- a/newapi/page.py +++ b/newapi/page.py @@ -1,13 +1,12 @@ -""" +""" """ -""" from .pages_bots.page import ( - home_dir, - MainPage, - user_agent, NEW_API, CatDepth, + MainPage, change_codes, + home_dir, + user_agent, ) __all__ = [ diff --git a/newapi/pages_bots/__init__.py b/newapi/pages_bots/__init__.py index d0add0e..6017056 100644 --- a/newapi/pages_bots/__init__.py +++ b/newapi/pages_bots/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os + if not os.getenv("BOTNAME"): botname = "newapi" os.environ["BOTNAME"] = botname diff --git a/newapi/pages_bots/all_apis.py b/newapi/pages_bots/all_apis.py index 86ba868..4e690dc 100644 --- a/newapi/pages_bots/all_apis.py +++ b/newapi/pages_bots/all_apis.py @@ -8,13 +8,15 @@ cat_members = main_api.CatDepth('Category Title') new_api = main_api.NEW_API() """ + # --- import functools from typing import Any + +from ..api_utils import printe from ..super.S_API import bot_api from ..super.S_Category import catdepth_new from ..super.S_Page import super_page -from ..api_utils import printe from ..super.super_login import Login @@ -52,12 +54,14 @@ def _login(self) -> Login: # --- login_bot = Login(self.lang, family=self.family) # --- - printe.output(f"### <> LoginWrap make new bot for ({self.lang}.{self.family}.org|{self.username})", p=True) + printe.output( + f"### <> LoginWrap make new bot for ({self.lang}.{self.family}.org|{self.username})", p=True + ) # --- user_tables = { self.family: { - 'username': self.username, - 'password': self.password, + "username": self.username, + "password": self.password, } } # --- @@ -67,5 +71,5 @@ def _login(self) -> Login: __all__ = [ - 'ALL_APIS', + "ALL_APIS", ] diff --git a/newapi/pages_bots/mdwiki_page.py b/newapi/pages_bots/mdwiki_page.py deleted file mode 100644 index cd56b09..0000000 --- a/newapi/pages_bots/mdwiki_page.py +++ /dev/null @@ -1,49 +0,0 @@ -# --- -""" -""" -# --- -import os -import functools -import sys - -if "mwclient" not in sys.argv: - sys.argv.append("nomwclient") - -from .all_apis import ALL_APIS -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes -from ..accounts.user_account_new import User_tables - -user_agent = default_user_agent() - -change_codes = lang_codes.change_codes - -home_dir = os.getenv("HOME") - - -@functools.lru_cache(maxsize=1) -def load_main_api() -> ALL_APIS: - return ALL_APIS( - lang="www", - family="mdwiki", - username=User_tables["username"], - password=User_tables["password"], - ) - - -main_api = load_main_api() - -NEW_API = main_api.NEW_API -MainPage = main_api.MainPage -CatDepth = main_api.CatDepth -md_MainPage = MainPage - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'md_MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots/ncc_page.py b/newapi/pages_bots/ncc_page.py deleted file mode 100644 index 9c92285..0000000 --- a/newapi/pages_bots/ncc_page.py +++ /dev/null @@ -1,96 +0,0 @@ -# --- -""" - -from newapi.ncc_page import CatDepth - -# cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, ns=10, nslist=[], onlyns=False, tempyes=[]) - -from newapi.ncc_page import MainPage as ncc_MainPage -''' -page = ncc_MainPage(title, 'www', family='nccommons') -exists = page.exists() -if not exists: return -# --- -''' - - -from newapi.ncc_page import NEW_API -# api_new = NEW_API('www', family='nccommons') -# login = api_new.Login_to_wiki() -# json1 = api_new.post_params(params, addtoken=False) -# pages = api_new.Find_pages_exists_or_not(liste) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# revisions= api_new.get_revisions(title) - -""" - -import os -# import sys -# --- -home_dir = os.getenv("HOME") -# --- -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes - -from ..accounts.user_account_ncc import User_tables, SITECODE, FAMILY -# --- -user_agent = default_user_agent() -# --- -change_codes = lang_codes.change_codes - -logins_cache = {} - -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - -def MainPage(title, lang, family=FAMILY): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - -def CatDepth(title, sitecode=SITECODE, family=FAMILY, **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result - - -ncc_MainPage = MainPage - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'ncc_MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots/new_wiki_pages.py b/newapi/pages_bots/new_wiki_pages.py deleted file mode 100644 index 2f098a0..0000000 --- a/newapi/pages_bots/new_wiki_pages.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -from newapi.page import MainPage, NEW_API, CatDepth - -Usage: - -from newapi.page import CatDepth - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], without_lang="", with_lang="", tempyes=[]) - -from newapi.page import MainPage, NEW_API -# api_new = NEW_API('en', family='wikipedia') -# json1 = api_new.post_params(params, addtoken=False) -# move_it = api_new.move(old_title, to, reason="", noredirect=False, movesubpages=False) -# pages = api_new.Find_pages_exists_or_not(liste, get_redirect=False) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# extlinks = api_new.get_extlinks(title) -# revisions= api_new.get_revisions(title) -# logs = api_new.get_logs(title) -# wantedcats = api_new.querypage_list(qppage='Wantedcategories', qplimit="max", Max=5000) -# pages = api_new.Get_template_pages(title, namespace="*", Max=10000) -""" -# --- -import functools -import os -import sys -from ..super.S_API import bot_api -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes -from .all_apis import ALL_APIS - -from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem -home_dir = os.getenv("HOME") -# --- -User_tables = User_tables_bot -# --- -if "workibrahem" in sys.argv: - User_tables = User_tables_ibrahem - # --- - print(f"page.py use {User_tables['username']} account.") - -user_agent = default_user_agent() -change_codes = lang_codes.change_codes -logins_cache = {} - - -@functools.lru_cache(maxsize=1) -def load_main_api(lang, family) -> ALL_APIS: - return ALL_APIS( - lang=lang, - family=family, - username=User_tables["username"], - password=User_tables["password"], - ) - - -def MainPage(title, lang, family="wikipedia"): - # --- - main_bot = load_main_api(lang, family) - # --- - page = main_bot.MainPage(title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode="", family="wikipedia", **kwargs): - # --- - main_bot = load_main_api(sitecode, family) - # --- - result = main_bot.CatDepth(title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia") -> bot_api.NEW_API: - main_bot = load_main_api(lang, family) - return main_bot.NEW_API() - - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots/page.py b/newapi/pages_bots/page.py index 7155955..23299fe 100644 --- a/newapi/pages_bots/page.py +++ b/newapi/pages_bots/page.py @@ -1,42 +1,16 @@ -""" -from newapi.page import MainPage, NEW_API, CatDepth +""" """ -Usage: - -from newapi.page import CatDepth - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], without_lang="", with_lang="", tempyes=[]) - -from newapi.page import MainPage, NEW_API -# api_new = NEW_API('en', family='wikipedia') -# json1 = api_new.post_params(params, addtoken=False) -# move_it = api_new.move(old_title, to, reason="", noredirect=False, movesubpages=False) -# pages = api_new.Find_pages_exists_or_not(liste, get_redirect=False) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# extlinks = api_new.get_extlinks(title) -# revisions= api_new.get_revisions(title) -# logs = api_new.get_logs(title) -# wantedcats = api_new.querypage_list(qppage='Wantedcategories', qplimit="max", Max=5000) -# pages = api_new.Get_template_pages(title, namespace="*", Max=10000) -""" # --- +import functools import os import sys -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent +from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem from ..api_utils import lang_codes +from ..api_utils.user_agent import default_user_agent +from ..super.S_API import bot_api +from .all_apis import ALL_APIS -from ..accounts.useraccount import User_tables_bot, User_tables_ibrahem home_dir = os.getenv("HOME") # --- User_tables = User_tables_bot @@ -45,55 +19,50 @@ User_tables = User_tables_ibrahem # --- print(f"page.py use {User_tables['username']} account.") -# --- + user_agent = default_user_agent() -# --- change_codes = lang_codes.change_codes - logins_cache = {} -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot +@functools.lru_cache(maxsize=1) +def load_main_api(lang, family) -> ALL_APIS: + return ALL_APIS( + lang=lang, + family=family, + username=User_tables["username"], + password=User_tables["password"], + ) def MainPage(title, lang, family="wikipedia"): # --- - login_bot = log_it(lang, family) + main_bot = load_main_api(lang, family) # --- - page = super_page.MainPage(login_bot, title, lang, family=family) + page = main_bot.MainPage(title, lang, family=family) # --- return page def CatDepth(title, sitecode="", family="wikipedia", **kwargs): # --- - login_bot = log_it(sitecode, family) + main_bot = load_main_api(sitecode, family) # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) + result = main_bot.CatDepth(title, sitecode=sitecode, family=family, **kwargs) # --- return result -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result +def NEW_API(lang="", family="wikipedia") -> bot_api.NEW_API: + main_bot = load_main_api(lang, family) + return main_bot.NEW_API() __all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', + "home_dir", + "user_agent", + "MainPage", + "NEW_API", + "CatDepth", + "change_codes", ] diff --git a/newapi/pages_bots/toolforge_page.py b/newapi/pages_bots/toolforge_page.py deleted file mode 100644 index 488dff7..0000000 --- a/newapi/pages_bots/toolforge_page.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -from newapi import toolforge_page -# --- -User_tables_md = { - "username": medwiki_account.username, - "password": medwiki_account.password, -} -# --- -toolforge_page.add_User_table(User_tables_md, "toolforge", "medwiki") -# --- -CatDepth = toolforge_page.subcatquery -MainPage = toolforge_page.MainPage(title, lang, family="toolforge") -# --- -""" -# --- -from ..super.S_Category import catdepth_new - -from ..super.S_API import bot_api -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap - -User_tables = {} -logins_cache = {} - - -def add_User_table(table, family, lang): - # --- - User_tables[(lang, family)] = table - User_tables[("*", family)] = table - - -def log_it(lang, family): - # --- - table = User_tables.get((lang, family)) or User_tables.get(("*", family)) - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, table) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - - -def MainPage(title, lang, family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode="", family="wikipedia", **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result - - -__all__ = [ - # "bot_api", - # "super_page", - "catdepth_new", - "MainPage", - "add_User_table", - "CatDepth", -] diff --git a/newapi/pages_bots/wiki_page.py b/newapi/pages_bots/wiki_page.py deleted file mode 100644 index 8cf6371..0000000 --- a/newapi/pages_bots/wiki_page.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Usage: - -from newapi.page import CatDepth - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], without_lang="", with_lang="", tempyes=[]) - -from newapi.page import MainPage, NEW_API -# api_new = NEW_API('en', family='wikipedia') -# json1 = api_new.post_params(params, addtoken=False) -# move_it = api_new.move(old_title, to, reason="", noredirect=False, movesubpages=False) -# pages = api_new.Find_pages_exists_or_not(liste, get_redirect=False) -# pages = api_new.Get_All_pages(start='', namespace="0", limit="max", apfilterredir='', limit_all=0) -# search = api_new.Search(value='', ns="", offset='', srlimit="max", RETURN_dict=False, addparams={}) -# newpages = api_new.Get_Newpages(limit="max", namespace="0", rcstart="", user='') -# usercont = api_new.UserContribs(user, limit=5000, namespace="*", ucshow="") -# l_links = api_new.Get_langlinks_for_list(titles, targtsitecode="", numbes=50) -# text_w = api_new.expandtemplates(text) -# subst = api_new.Parse_Text('{{subst:page_name}}', title) -# extlinks = api_new.get_extlinks(title) -# revisions= api_new.get_revisions(title) -# logs = api_new.get_logs(title) -# wantedcats = api_new.querypage_list(qppage='Wantedcategories', qplimit="max", Max=5000) -# pages = api_new.Get_template_pages(title, namespace="*", Max=10000) -""" -# --- -import os -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new - -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes - -from ..accounts.user_account_new import User_tables_wiki - -home_dir = os.getenv("HOME") -# --- -User_tables = User_tables_wiki -# --- -print(f"page.py use {User_tables['username']} account.") -# --- -user_agent = default_user_agent() -# --- -change_codes = lang_codes.change_codes - -logins_cache = {} - - -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - - -def MainPage(title, lang, family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode="", family="wikipedia", **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia"): - # --- - login_bot = log_it(lang, family) - # --- - result = bot_api.NEW_API(login_bot, lang=lang, family=family) - # --- - return result - - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pages_bots_backup/mdwiki_page.py b/newapi/pages_bots_backup/mdwiki_page.py deleted file mode 100644 index 5ae5b36..0000000 --- a/newapi/pages_bots_backup/mdwiki_page.py +++ /dev/null @@ -1,106 +0,0 @@ -# --- -""" - -# cat_members = CatDepth(title, sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], onlyns=False, without_lang="", with_lang="", tempyes=[]) - -''' -page = MainPage(title, 'www', family='mdwiki') -exists = page.exists() -if not exists: return -# --- -page_edit = page.can_edit() -if not page_edit: return -# --- -if page.isRedirect() : return -# target = page.get_redirect_target() -# --- -text = page.get_text() -ns = page.namespace() -links = page.page_links() -categories = page.get_categories(with_hidden=False) -langlinks = page.get_langlinks() -wiki_links = page.get_wiki_links_from_text() -refs = page.Get_tags(tag='ref')# for x in ref: name, contents = x.name, x.contents -words = page.get_words() -templates = page.get_templates() -save_page = page.save(newtext='', summary='', nocreate=1, minor='') -create = page.Create(text='', summary='') -# --- -back_links = page.page_backlinks() -text_html = page.get_text_html() -hidden_categories= page.get_hidden_categories() -flagged = page.is_flagged() -timestamp = page.get_timestamp() -user = page.get_user() -purge = page.purge() -''' -""" -# --- -import os -import sys -# --- -home_dir = os.getenv("HOME") -# --- -if "mwclient" not in sys.argv: - sys.argv.append("nomwclient") - # print("sys.argv.append('nomwclient')") - -from ..super.S_API import bot_api -from ..super.S_Category import catdepth_new -from ..super.S_Page import super_page -from ..super.login_wrap import LoginWrap -from ..api_utils.user_agent import default_user_agent -from ..api_utils import lang_codes - -from ..accounts.user_account_new import User_tables, SITECODE, FAMILY -# --- -user_agent = default_user_agent() -# --- -change_codes = lang_codes.change_codes - -logins_cache = {} - - -def log_it(lang, family): - # --- - login_bot, logins_cache2 = LoginWrap(lang, family, logins_cache, User_tables) - # --- - logins_cache.update(logins_cache2) - # --- - return login_bot - - -def MainPage(title, lang, family=FAMILY): - # --- - login_bot = log_it(lang, family) - # --- - page = super_page.MainPage(login_bot, title, lang, family=family) - # --- - return page - - -def CatDepth(title, sitecode=SITECODE, family=FAMILY, **kwargs): - # --- - login_bot = log_it(sitecode, family) - # --- - result = catdepth_new.subcatquery(login_bot, title, sitecode=sitecode, family=family, **kwargs) - # --- - return result - - -def NEW_API(lang="", family="wikipedia"): - login_bot = log_it(lang, family) - return bot_api.NEW_API(login_bot, lang=lang, family=family) - - -md_MainPage = MainPage - -__all__ = [ - 'home_dir', - 'user_agent', - 'MainPage', - 'md_MainPage', - 'NEW_API', - 'CatDepth', - 'change_codes', -] diff --git a/newapi/pformat.py b/newapi/pformat.py index d29a918..61a124f 100644 --- a/newapi/pformat.py +++ b/newapi/pformat.py @@ -1,10 +1,11 @@ """ python3 core8/pwb.py newapi/pformat """ + import sys -import wikitextparser as wtp from pathlib import Path +import wikitextparser as wtp # python3 core8/pwb.py newapi/pformat -title:قالب:Cycling_race/stageclassification3 # python3 core8/pwb.py newapi/pformat -title:قالب:معلومات_خاصية_ويكي_بيانات/تتبع @@ -27,39 +28,3 @@ def make_new_text(text): new_text = new_text.replace(temp_text, new_temp) # --- return new_text - - -if __name__ == "__main__": - # --- - sys.argv.append('workibrahem') - sys.argv.append('ask') - # --- - Dir = Path(__file__).parent - # --- - title = '' - text = open(f"{Dir}/pformat.txt", "r", encoding="utf-8").read() - for arg in sys.argv: - arg, _, value = arg.partition(':') - if arg == '-title' or arg == '-page': - title = value - # --- - if title: - # --- - from newapi.page import MainPage - - page = MainPage(title, 'ar', family='wikipedia') - text = page.get_text() - newtext = make_new_text(text) - if 'save' in sys.argv: - save_page = page.save(newtext=newtext, summary='', nocreate=1, minor='') - else: - prased = wtp.parse(text) - newtext = prased.pformat() - # --- - print(newtext) - # --- - with open(f"{Dir}/pformat.txt", "w", encoding="utf-8") as logfile: - logfile.write(newtext) - # --- - print('add "save" to sys.argv to save.') - # --- diff --git a/newapi/printe.py b/newapi/printe.py index 2cf9914..7d75fbc 100644 --- a/newapi/printe.py +++ b/newapi/printe.py @@ -1,14 +1,13 @@ -""" +""" """ -""" from .api_utils.printe import ( - showDiff, - output, debug, - warn, error, info, + output, + showDiff, test_print, + warn, ) __all__ = [ diff --git a/newapi/super/Login_db/__init__.py b/newapi/super/Login_db/__init__.py deleted file mode 100644 index 7c68785..0000000 --- a/newapi/super/Login_db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/newapi/super/Login_db/bot.py b/newapi/super/Login_db/bot.py deleted file mode 100644 index e2f0882..0000000 --- a/newapi/super/Login_db/bot.py +++ /dev/null @@ -1,165 +0,0 @@ -""" -Module for handling login attempt logging to database - -from newapi.super.Login_db.bot2 import log_one - -# log_one(site="", user="", result="", db_type="sqlite", db_path="login_logs.db", credentials=None) - -""" -import functools -import sys -import os -import datetime -from configparser import ConfigParser -from ...api_utils import printe -from ...DB_bots.pymysql_bot import sql_connect_pymysql - -LOGS_IS_ENABLED = os.getenv("LOGIN_LOGS_IS_ENABLED", "0") == "1" - - -def local_host(): - main_args = { - "host": "127.0.0.1", - "db": "mv", - "charset": "utf8mb4", - # "cursorclass": Typee, - "use_unicode": True, - "autocommit": True, - } - - credentials = {"user": "root", "password": "root11"} - - return main_args, credentials - - -@functools.lru_cache(maxsize=1) -def jls_extract_def(): - # --- - if not os.getenv("HOME"): - return local_host() - # --- - project = os.getenv("HOME") - # --- - if __file__.find('/data/project/himo') != -1: - project = "/data/project/himo" - # --- - db_connect_file = f"{project}/confs/db.ini" - db_username = "" - credentials = {"read_default_file": db_connect_file} - # --- - # if db_connect_file is not file - if os.path.isfile(db_connect_file): - # --- - try: - config2 = ConfigParser() - config2.read(db_connect_file) - db_username = config2["client"]["user"] - except KeyError as e: - print(f"error: {e}") - # --- - # Typee = pymysql.cursors.DictCursor if return_dict else pymysql.cursors.Cursor - # --- - main_args = { - "host": "tools.db.svc.wikimedia.cloud", - "db": f"{db_username}__mv", - "charset": "utf8mb4", - # "cursorclass": Typee, - "use_unicode": True, - "autocommit": True, - } - - if os.getenv("HOME").find("/data/project/mdwiki") != -1: - main_args["db"] = f"{db_username}__mdwiki_new" - - return main_args, credentials - - -main_args, credentials = jls_extract_def() - -userinfo = 0 - - -def log_one(site, user, result, action="", params={}) -> None: - # global userinfo - # --- - if not LOGS_IS_ENABLED: - return None - # --- - params = params or {} - # --- - if action == "query": - if params.get('meta', "") == "tokens": - action = "tokens" - if params.get('type'): - action += "_" + params['type'] - elif params.get('meta', "").find("userinfo") != -1: - action = "userinfo" - # --- - # if params.get('meta', "").find("userinfo") != -1: - # userinfo += 1 - # if userinfo > 3: - # raise Exception("too much") - # --- - if action == "query" and params.get('prop'): - action += "_" + "_".join(params['prop'].split("|")) - # --- - if action == "query" and "test" in sys.argv: - printe.output(f"<> {action=}") - print(dict(params)) - # --- - # Create table query - _create_table_query = """ - CREATE TABLE IF NOT EXISTS login_attempts ( - id INT AUTO_INCREMENT PRIMARY KEY, - site VARCHAR(255) NOT NULL, - username VARCHAR(255) NOT NULL, - result VARCHAR(50) NOT NULL, - timestamp DATETIME NOT NULL, - action VARCHAR(100) NULL, - INDEX idx_site_user (site, username), - INDEX idx_timestamp (timestamp) - ) - """ - # Create table query - create_table_query = """ - CREATE TABLE IF NOT EXISTS `logins` ( - `id` int NOT NULL AUTO_INCREMENT, - `site` varchar(255) NOT NULL, - `username` varchar(255) NOT NULL, - `result` varchar(50) NOT NULL, - `action` varchar(100) DEFAULT NULL, - `count` int NOT NULL DEFAULT '1', - `first` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `last` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `site_username_result_action` (`site`,`username`,`result`,`action`), - KEY `idx_site_user` (`site`,`username`), - KEY `idx_timestamp` (`first`) - ) - """ - # print(f"create_table_query: {create_table_query}") - - timestamp = datetime.datetime.now().isoformat() - - # Execute table creation - sql_connect_pymysql( - create_table_query, - main_args=main_args, - credentials=credentials - ) - - # Insert login attempt - insert_query = """ - INSERT INTO logins (site, username, result, action, count, first, last) - VALUES (%s, %s, %s, %s, %s, NOW(), NOW()) - ON DUPLICATE KEY UPDATE count = count + 1, last = NOW() - """ - - # print(f"insert_query: {insert_query}") - - sql_connect_pymysql( - insert_query, - values=(site, user, result, action, 1), - main_args=main_args, - credentials=credentials, - ) diff --git a/newapi/super/Login_db/bot1.py b/newapi/super/Login_db/bot1.py deleted file mode 100644 index e248119..0000000 --- a/newapi/super/Login_db/bot1.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Module for handling login attempt logging to database - -from newapi.super.Login_db.bot import log_one - -# log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=login_result) - -""" -import sys -import os -import json -import pymysql -import datetime - -DB_CONFIG = { - 'host': 'localhost', - 'user': 'root', - 'password': 'password', - 'database': 'login_db', - 'charset': 'utf8mb4', - 'cursorclass': pymysql.cursors.DictCursor -} - -LOGS_IS_ENABLED = os.getenv("LOGIN_LOGS_IS_ENABLED", "0") == "1" - - -def log_one(site="", user="", result=""): - """ - Logs a login attempt to the database - - Args: - site (str): The site where the login attempt occurred - user (str): The username used in the attempt - result (str): The result of the attempt (success/failure) - """ - # --- - if not LOGS_IS_ENABLED: - return - # --- - timestamp = datetime.datetime.now().isoformat() - # --- - insert_query = """ - INSERT INTO login_attempts (site, username, result, timestamp) - VALUES (%s, %s, %s, %s) - """ - # --- - try: - connection = pymysql.connect(**DB_CONFIG) - try: - with connection.cursor() as cursor: - cursor.execute(insert_query, (site, user, result, timestamp)) - connection.commit() - except Exception as e: - connection.rollback() - print(f"Database error occurred: {e}") - finally: - connection.close() - except pymysql.Error as e: - print(f"Database error occurred: {e}") diff --git a/newapi/super/Login_db/lite.py b/newapi/super/Login_db/lite.py deleted file mode 100644 index 3c52ea1..0000000 --- a/newapi/super/Login_db/lite.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Module for handling login attempt logging to database - -from newapi.super.Login_db.bot2 import log_one - -# log_one(site="", user="", result="", db_type="sqlite", db_path="login_logs.db", credentials=None) - -""" -from ...api_utils.except_err import exception_err -import datetime -from pathlib import Path -from ...DB_bots import db_bot -LiteDB = db_bot.LiteDB - -def log_one_sqlite(site="", user="", result=""): - Dir = Path(__file__).resolve().parent - - db_path = Dir / "login_logs.db" - - timestamp = datetime.datetime.now().isoformat() - - try: - db = LiteDB(db_path) - - # Create table if it doesn't exist - db.create_table( - "login_attempts", - { - "id": int, - "site": str, - "username": str, - "result": str, - "timestamp": str - }, - pk="id" - ) - - # Insert login attempt - db.insert("login_attempts", { - "site": site, - "username": user, - "result": result, - "timestamp": timestamp - }, check=False) # Don't check for duplicates since each attempt should be logged - - except Exception as e: - exception_err(e, "Failed to log login attempt to SQLite") diff --git a/newapi/super/S_API/bot.py b/newapi/super/S_API/bot.py index 1a17600..35cabb6 100644 --- a/newapi/super/S_API/bot.py +++ b/newapi/super/S_API/bot.py @@ -3,10 +3,12 @@ from .super.S_API.bot import BOTS_APIS """ + import sys + from ...api_utils import printe -from ..handel_errors import HANDEL_ERRORS from ...api_utils.ask_bot import ASK_BOT +from ..handel_errors import HANDEL_ERRORS yes_answer = ["y", "a", "", "Y", "A", "all", "aaa"] file_name = "bot_api.py" @@ -33,9 +35,15 @@ def Add_To_Bottom(self, text, summary, title, poss="Head|Bottom"): printe.test_print(f"** Add_To_Bottom .. [[{title}]] ") # printe.showDiff("", text) # --- - user = self.username or getattr(self, 'user_login', '') + user = self.username or getattr(self, "user_login", "") # --- - ask = self.ask_put(newtext=text, message=f"** Add_To {poss} .. [[{title}]] ", job="Add_To_Bottom", username=user, summary=summary) + ask = self.ask_put( + newtext=text, + message=f"** Add_To {poss} .. [[{title}]] ", + job="Add_To_Bottom", + username=user, + summary=summary, + ) # --- if ask is False: return False @@ -104,7 +112,7 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r # --- message = f"Do you want to move page:[[{old_title}]] to [[{to}]]?" # --- - user = self.username or getattr(self, 'user_login', '') + user = self.username or getattr(self, "user_login", "") # --- if not self.ask_put(message=message, job="move", username=user): return {} @@ -123,9 +131,30 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r "reason": "wrong title", "redirectcreated": True, "moveoverredirect": False, - "talkmove-errors": [{"message": "content-not-allowed-here", "params": ["Structured Discussions board", "User talk:Mr. Ibrahem/x", "main"], "code": "contentnotallowedhere", "type": "error"}, {"message": "flow-error-allowcreation-flow-create-board", "params": [], "code": "flow-error-allowcreation-flow-create-board", "type": "error"}], - "subpages": {"errors": [{"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"}]}, - "subpages-talk": {"errors": [{"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"}]}, + "talkmove-errors": [ + { + "message": "content-not-allowed-here", + "params": ["Structured Discussions board", "User talk:Mr. Ibrahem/x", "main"], + "code": "contentnotallowedhere", + "type": "error", + }, + { + "message": "flow-error-allowcreation-flow-create-board", + "params": [], + "code": "flow-error-allowcreation-flow-create-board", + "type": "error", + }, + ], + "subpages": { + "errors": [ + {"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"} + ] + }, + "subpages-talk": { + "errors": [ + {"message": "cant-move-subpages", "params": [], "code": "cant-move-subpages", "type": "error"} + ] + }, } } # --- @@ -146,7 +175,14 @@ def move(self, old_title, to, reason="", noredirect=False, movesubpages=False, r if error: if error_code == "ratelimited": printe.output("<> move ratelimited:") - return self.move(old_title, to, reason=reason, noredirect=noredirect, movesubpages=movesubpages, return_dict=return_dict) + return self.move( + old_title, + to, + reason=reason, + noredirect=noredirect, + movesubpages=movesubpages, + return_dict=return_dict, + ) if error_code == "articleexists": printe.output("<> articleexists") diff --git a/newapi/super/S_API/bot_api.py b/newapi/super/S_API/bot_api.py index 1d31a3e..b43478c 100644 --- a/newapi/super/S_API/bot_api.py +++ b/newapi/super/S_API/bot_api.py @@ -40,16 +40,19 @@ api_new = NEW_API(code, family='wikipedia') api_new.Login_to_wiki() """ -# --- -import tqdm -import time + +import datetime import sys +import time from collections.abc import KeysView -import datetime from datetime import timedelta + +# --- +import tqdm + from ...api_utils import printe -from .bot import BOTS_APIS from ...api_utils.lang_codes import change_codes +from .bot import BOTS_APIS class NEW_API(BOTS_APIS): @@ -74,25 +77,14 @@ def __init__(self, login_bot, lang="", family="wikipedia"): def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): # --- return self.login_bot.post_params( - params, - Type=Type, - addtoken=addtoken, - GET_CSRF=GET_CSRF, - files=files, - do_error=do_error, - max_retry=max_retry + params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry ) - def post_continue(self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None): + def post_continue( + self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + ): return self.login_bot.post_continue( - params, - action, - _p_=_p_, - p_empty=p_empty, - Max=Max, - first=first, - _p_2=_p_2, - _p_2_empty=_p_2_empty + params, action, _p_=_p_, p_empty=p_empty, Max=Max, first=first, _p_2=_p_2, _p_2_empty=_p_2_empty ) def get_username(self): @@ -140,7 +132,7 @@ def Find_pages_exists_or_not(self, liste, get_redirect=False, noprint=False): query_table = all_jsons.get("query", {}) # --- normalz = query_table.get("normalized", []) - normalized = {red["to"] : red["from"] for red in normalz} + normalized = {red["to"]: red["from"] for red in normalz} # --- query_pages = query_table.get("pages", []) # --- @@ -174,7 +166,15 @@ def Find_pages_exists_or_not(self, liste, get_redirect=False, noprint=False): # --- return table - def Find_pages_exists_or_not_with_qids(self, liste, get_redirect=False, noprint=False, return_all_jsons=False, use_user_input_title=False, chunk_size=50): + def Find_pages_exists_or_not_with_qids( + self, + liste, + get_redirect=False, + noprint=False, + return_all_jsons=False, + use_user_input_title=False, + chunk_size=50, + ): # --- done = 0 # --- @@ -240,10 +240,7 @@ def Find_pages_exists_or_not_with_qids(self, liste, get_redirect=False, noprint= if use_user_input_title and title_tab.get("user_input"): title_x = title_tab["user_input"] # --- - table.setdefault(title_x, { - "wikibase_item": wikibase_item, - "exist": False - }) + table.setdefault(title_x, {"wikibase_item": wikibase_item, "exist": False}) # --- if title_tab: table[title_x]["title_tab"] = title_tab @@ -272,7 +269,9 @@ def Find_pages_exists_or_not_with_qids(self, liste, get_redirect=False, noprint= def Get_All_pages(self, start="", namespace="0", limit="max", apfilterredir="", ppprop="", limit_all=100000): # --- - printe.test_print(f"Get_All_pages for start:{start}, limit:{limit},namespace:{namespace},apfilterredir:{apfilterredir}") + printe.test_print( + f"Get_All_pages for start:{start}, limit:{limit},namespace:{namespace},apfilterredir:{apfilterredir}" + ) # --- params = { "action": "query", @@ -368,9 +367,13 @@ def PrefixSearch(self, pssearch="", ns="0", pslimit="max", limit_all=100000): # --- return Main_table - def Get_All_pages_generator(self, start="", namespace="0", limit="max", filterredir="", ppprop="", limit_all=100000): + def Get_All_pages_generator( + self, start="", namespace="0", limit="max", filterredir="", ppprop="", limit_all=100000 + ): # --- - printe.test_print(f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}") + printe.test_print( + f"Get_All_pages_generator for start:{start}, limit:{limit},namespace:{namespace},filterredir:{filterredir}" + ) # --- params = { "action": "query", @@ -449,7 +452,16 @@ def Search(self, value="", ns="*", offset="", srlimit="max", RETURN_dict=False, # --- return results - def Get_Newpages(self, limit=5000, namespace="0", rcstart="", user="", three_houers=False, offset_minutes=False, offset_hours=False): + def Get_Newpages( + self, + limit=5000, + namespace="0", + rcstart="", + user="", + three_houers=False, + offset_minutes=False, + offset_hours=False, + ): if three_houers: dd = datetime.datetime.utcnow() - timedelta(hours=3) rcstart = dd.strftime("%Y-%m-%dT%H:%M:00.000Z") @@ -617,7 +629,9 @@ def Get_langlinks_for_list(self, titles, targtsitecode="", numbes=40): if lang["lang"] == targtsitecode: find_targtsitecode += 1 # --- - printe.output(f'bot_api.Get_langlinks_for_list find "{len(table)}" in table,find_targtsitecode:{targtsitecode}:{find_targtsitecode}') + printe.output( + f'bot_api.Get_langlinks_for_list find "{len(table)}" in table,find_targtsitecode:{targtsitecode}:{find_targtsitecode}' + ) # --- return table @@ -933,10 +947,22 @@ def users_infos(self, ususers=[]): "utf8": 1, "formatversion": "2", "usprop": "groups|implicitgroups|editcount|gender|registration", - "ususers": "Mr.Ibrahembot" + "ususers": "Mr.Ibrahembot", } # --- - all_usprops = ["groups", "implicitgroups", "cancreate", "editcount", "centralids", "blockinfo", "emailable", "gender", "groupmemberships", "registration", "rights"] + all_usprops = [ + "groups", + "implicitgroups", + "cancreate", + "editcount", + "centralids", + "blockinfo", + "emailable", + "gender", + "groupmemberships", + "registration", + "rights", + ] # --- ususers = list(set(ususers)) # --- diff --git a/newapi/super/S_Category/bot.py b/newapi/super/S_Category/bot.py index e3840cd..b31ac9c 100644 --- a/newapi/super/S_Category/bot.py +++ b/newapi/super/S_Category/bot.py @@ -3,7 +3,9 @@ from .bot import CategoryDepth """ + import copy + import tqdm from ...api_utils import printe @@ -67,13 +69,7 @@ def __init__(self, login_bot, title, **kwargs): def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): # --- return self.login_bot.post_params( - params, - Type=Type, - addtoken=addtoken, - GET_CSRF=GET_CSRF, - files=files, - do_error=do_error, - max_retry=max_retry + params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry ) def get_revids(self): @@ -347,7 +343,9 @@ def subcatquery_(self, **kwargs): new_tab2 = [] # --- if self.limit > 0 and len(self.result_table) >= self.limit: - printe.output(f"<> limit:{self.limit} reached, len of results: {len(self.result_table)} break ..") + printe.output( + f"<> limit:{self.limit} reached, len of results: {len(self.result_table)} break .." + ) break # --- depth_done += 1 diff --git a/newapi/super/S_Category/catdepth_new.py b/newapi/super/S_Category/catdepth_new.py index fae05e4..82ad3b0 100644 --- a/newapi/super/S_Category/catdepth_new.py +++ b/newapi/super/S_Category/catdepth_new.py @@ -1,11 +1,7 @@ -""" +""" """ -from newapi.ncc_page import CatDepth -# cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, ns=10, nslist=[], onlyns=False, tempyes=[]) - -""" -import time import sys +import time from functools import lru_cache from ...api_utils import printe @@ -40,7 +36,7 @@ def args_group(title, kwargs): "with_lang": None, "tempyes": None, "props": None, - "only_titles": None + "only_titles": None, } # --- for k, v in kwargs.items(): @@ -61,7 +57,9 @@ def subcatquery(login_bot, title, sitecode=SITECODE, family=FAMILY, **kwargs): args2 = args_group(title, kwargs) # --- if print_s: - printe.output(f"<> catdepth_new.py sub cat query for {sitecode}:{title}, depth:{args2['depth']}, ns:{args2['ns']}, onlyns:{args2['onlyns']}") + printe.output( + f"<> catdepth_new.py sub cat query for {sitecode}:{title}, depth:{args2['depth']}, ns:{args2['ns']}, onlyns:{args2['onlyns']}" + ) # --- start = time.time() # --- @@ -80,6 +78,8 @@ def subcatquery(login_bot, title, sitecode=SITECODE, family=FAMILY, **kwargs): if print_s: lenpages = bot.get_len_pages() # --- - printe.output(f"<>catdepth_new.py: find {len(result)} pages({args2['ns']}) in {sitecode}:{title}, depth:{args2['depth']} in {delta} seconds | {lenpages=}") + printe.output( + f"<>catdepth_new.py: find {len(result)} pages({args2['ns']}) in {sitecode}:{title}, depth:{args2['depth']} in {delta} seconds | {lenpages=}" + ) # --- return result diff --git a/newapi/super/S_Page/ar_err.py b/newapi/super/S_Page/ar_err.py index c4f5d82..2c680c3 100644 --- a/newapi/super/S_Page/ar_err.py +++ b/newapi/super/S_Page/ar_err.py @@ -40,8 +40,3 @@ def test_find_edit_error(): print(f"Test case 3: Result = {result}") print("All test cases pass!") - - -if __name__ == "__main__": - # Run the test - test_find_edit_error() diff --git a/newapi/super/S_Page/bot.py b/newapi/super/S_Page/bot.py index 753004d..9607088 100644 --- a/newapi/super/S_Page/bot.py +++ b/newapi/super/S_Page/bot.py @@ -3,8 +3,10 @@ from .super.S_Page.bot import PAGE_APIS """ + from ..handel_errors import HANDEL_ERRORS + class PAGE_APIS(HANDEL_ERRORS): def __init__(self, login_bot): # print("class PAGE_APIS:") @@ -16,14 +18,9 @@ def __init__(self, login_bot): # --- super().__init__() - def post_continue(self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None): + def post_continue( + self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + ): return self.login_bot.post_continue( - params, - action, - _p_=_p_, - p_empty=p_empty, - Max=Max, - first=first, - _p_2=_p_2, - _p_2_empty=_p_2_empty + params, action, _p_=_p_, p_empty=p_empty, Max=Max, first=first, _p_2=_p_2, _p_2_empty=_p_2_empty ) diff --git a/newapi/super/S_Page/data.py b/newapi/super/S_Page/data.py index a6893df..cc95813 100644 --- a/newapi/super/S_Page/data.py +++ b/newapi/super/S_Page/data.py @@ -20,6 +20,8 @@ self\.(text_html|words|length|summary) self.content.$1 """ + + @dataclass class Content: # text: str = "" @@ -29,6 +31,7 @@ class Content: words: int = 0 length: int = 0 + r""" # meta self.is_Disambig = False @@ -45,6 +48,8 @@ class Content: self\.(is_Disambig|can_be_edit|userinfo|create_data|info|username|Exists|is_redirect|flagged|wikibase_item) self.meta.$1 """ + + @dataclass class Meta: is_Disambig: bool = False @@ -59,6 +64,7 @@ class Meta: flagged: str = "" wikibase_item: str = "" + r""" # revisions_data self.revid = "" @@ -72,6 +78,7 @@ class Meta: self.revisions_data.$1 """ + @dataclass class RevisionsData: revid: str = "" @@ -81,6 +88,7 @@ class RevisionsData: revisions: list = field(default_factory=list) touched: str = "" + r""" # LinksData self.back_links = [] @@ -93,6 +101,8 @@ class RevisionsData: self\.(back_links|links_here|extlinks|links|iwlinks) self.links_data.$1 """ + + @dataclass class LinksData: back_links: list = field(default_factory=list) @@ -112,12 +122,15 @@ class LinksData: self\.(categories|hidden_categories|all_categories_with_hidden) self.categories_data.$1 """ + + @dataclass class CategoriesData: categories: dict = field(default_factory=dict) hidden_categories: dict = field(default_factory=dict) all_categories_with_hidden: dict = field(default_factory=dict) + r""" # TemplateData self.templates = {} @@ -126,6 +139,8 @@ class CategoriesData: self\.(templates_API|templates) self.template_data.$1 """ + + @dataclass class TemplateData: templates: dict = field(default_factory=dict) diff --git a/newapi/super/S_Page/super_page.py b/newapi/super/S_Page/super_page.py index 0be33eb..8d7e360 100644 --- a/newapi/super/S_Page/super_page.py +++ b/newapi/super/S_Page/super_page.py @@ -43,19 +43,20 @@ ''' """ + import os -from warnings import warn import sys -import wikitextparser as wtp +from warnings import warn -from .ar_err import find_edit_error -from .bot import PAGE_APIS -from .data import Content, Meta, RevisionsData, LinksData, CategoriesData, TemplateData +import wikitextparser as wtp -from ...api_utils import printe, txtlib, botEdit -from ...api_utils.except_err import exception_err, warn_err +from ...api_utils import botEdit, printe, txtlib from ...api_utils.ask_bot import ASK_BOT +from ...api_utils.except_err import exception_err, warn_err from ...api_utils.lang_codes import change_codes +from .ar_err import find_edit_error +from .bot import PAGE_APIS +from .data import CategoriesData, Content, LinksData, Meta, RevisionsData, TemplateData print_test = {1: "test" in sys.argv} @@ -98,13 +99,7 @@ def __init__(self, login_bot, title, lang="", family="wikipedia"): def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=None, do_error=False, max_retry=0): # --- return self.login_bot.post_params( - params, - Type=Type, - addtoken=addtoken, - GET_CSRF=GET_CSRF, - files=files, - do_error=do_error, - max_retry=max_retry + params, Type=Type, addtoken=addtoken, GET_CSRF=GET_CSRF, files=files, do_error=do_error, max_retry=max_retry ) def false_edit(self): @@ -182,7 +177,7 @@ def find_create_data(self): "rvprop": "timestamp|ids|user", "rvslots": "*", "rvlimit": "1", - "rvdir": "newer" + "rvdir": "newer", } # --- data = self.post_params(params) @@ -195,9 +190,9 @@ def find_create_data(self): # --- if "parentid" in page_data and page_data["parentid"] == 0: self.meta.create_data = { - "timestamp" : page_data["timestamp"], - "user" : page_data.get("user", ""), - "anon" : page_data.get("anon", False), + "timestamp": page_data["timestamp"], + "user": page_data.get("user", ""), + "anon": page_data.get("anon", False), } # --- break @@ -267,9 +262,9 @@ def get_text(self, redirects=False): # --- if "parentid" in page_data and page_data["parentid"] == 0: self.meta.create_data = { - "timestamp" : page_data["timestamp"], - "user" : page_data.get("user", ""), - "anon" : page_data.get("anon", False), + "timestamp": page_data["timestamp"], + "user": page_data.get("user", ""), + "anon": page_data.get("anon", False), } # --- break @@ -341,15 +336,15 @@ def get_infos(self): if "sortkey" in cat: del cat["sortkey"] # --- - tit = cat["title"] + category_title = cat["title"] # --- - self.categories_data.all_categories_with_hidden[tit] = cat + self.categories_data.all_categories_with_hidden[category_title] = cat # --- if cat.get("hidden") is True: - self.categories_data.hidden_categories[tit] = cat + self.categories_data.hidden_categories[category_title] = cat else: del cat["hidden"] - self.categories_data.categories[tit] = cat + self.categories_data.categories[category_title] = cat # --- if ta.get("langlinks", []) != []: # --- @@ -599,7 +594,9 @@ def can_edit(self, script="", delay=0): if not self.text: self.text = self.get_text() # --- - self.meta.can_be_edit = botEdit.bot_May_Edit(text=self.text, title_page=self.title, botjob=script, page=self, delay=delay) + self.meta.can_be_edit = botEdit.bot_May_Edit( + text=self.text, title_page=self.title, botjob=script, page=self, delay=delay + ) # --- return self.meta.can_be_edit @@ -703,9 +700,20 @@ def save(self, newtext="", summary="", nocreate=1, minor="0", tags="", nodiff=Fa # --- message = f"Do you want to save this page? ({self.lang}:{self.title})" # --- - user = self.meta.username or getattr(self, 'user_login', '') - # --- - if self.ask_put(nodiff=nodiff, newtext=newtext, text=self.text, message=message, job="save", username=user, summary=self.content.summary) is False: + user = self.meta.username or getattr(self, "user_login", "") + # --- + if ( + self.ask_put( + nodiff=nodiff, + newtext=newtext, + text=self.text, + message=message, + job="save", + username=user, + summary=self.content.summary, + ) + is False + ): return False # --- params = { @@ -821,9 +829,12 @@ def Create(self, text="", summary="", nodiff="", noask=False): # --- message = f"Do you want to create this page? ({self.lang}:{self.title})" # --- - user = self.meta.username or getattr(self, 'user_login', '') + user = self.meta.username or getattr(self, "user_login", "") # --- - if self.ask_put(nodiff=nodiff, newtext=text, message=message, job="create", username=user, summary=summary) is False: + if ( + self.ask_put(nodiff=nodiff, newtext=text, message=message, job="create", username=user, summary=summary) + is False + ): return False # --- params = { @@ -873,6 +884,9 @@ def Create(self, text="", summary="", nodiff="", noask=False): # --- return False + def create(self, text="", summary="", nodiff="", noask=False): + return self.Create(text=text, summary=summary, nodiff=nodiff, noask=noask) + def page_backlinks(self, ns=0): params = { "action": "query", diff --git a/newapi/super/__init__.py b/newapi/super/__init__.py index 2793e85..128ed0f 100644 --- a/newapi/super/__init__.py +++ b/newapi/super/__init__.py @@ -1,16 +1,14 @@ # -*- coding: utf-8 -*- -""" - -""" -from .S_API import bot_api +""" """ from . import super_login -from .S_Page import super_page +from .S_API import bot_api from .S_Category import catdepth_new +from .S_Page import super_page __all__ = [ - 'S_API', - 'bot_api', - 'super_page', - 'super_login', - 'catdepth_new', + "S_API", + "bot_api", + "super_page", + "super_login", + "catdepth_new", ] diff --git a/newapi/super/bot.py b/newapi/super/bot.py index 60daaa4..fc3b8dd 100644 --- a/newapi/super/bot.py +++ b/newapi/super/bot.py @@ -6,25 +6,25 @@ Exception:{'login': {'result': 'Failed', 'reason': 'You have made too many recent login attempts. Please wait 5 minutes before trying again.'}} """ -import sys -import os -import requests - -from .params_help import PARAMS_HELPS +import os +import logging +import sys from http.cookiejar import MozillaCookieJar +import requests + from ..api_utils import printe -from .cookies_bot import get_file_name, del_cookies_file from ..api_utils.except_err import exception_err -from .Login_db.bot import log_one from ..api_utils.user_agent import default_user_agent +from .cookies_bot import del_cookies_file, get_file_name +from .params_help import PARAMS_HELPS # cookies = get_cookies(lang, family, username) seasons_by_lang = {} users_by_lang = {} logins_count = {1: 0} - +logger = logging.getLogger(__name__) botname = "newapi" @@ -58,7 +58,7 @@ def __init__(self) -> None: super().__init__() def log_error(self, result, action, params=None) -> None: - log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=result, action=action, params=params) + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def add_User_tables(self, family, table, lang="") -> None: # --- @@ -71,7 +71,7 @@ def add_User_tables(self, family, table, lang="") -> None: if table["username"].find("bot") == -1 and family == "wikipedia": print(f"add_User_tables: {family=}, {table['username']=}") # --- - if family != "" and table['username'] != "" and table['password'] != "": + if family != "" and table["username"] != "" and table["password"] != "": # --- if self.family == family or (langx == "ar" and self.family.startswith("wik")): # wiktionary self.user_table_done = True @@ -111,7 +111,9 @@ def log_in(self) -> bool: Bot_passwords = self.password.find("@") != -1 logins_count[1] += 1 printe.output(f"<<{color}>> {botname}/page.py: Log_to_wiki {self.endpoint} count:{logins_count[1]}") - printe.output(f"{botname}/page.py: log to {self.lang}.{self.family}.org user:{self.username}, ({Bot_passwords=})") + printe.output( + f"{botname}/page.py: log to {self.lang}.{self.family}.org user:{self.username}, ({Bot_passwords=})" + ) logintoken = self.get_logintoken() @@ -142,7 +144,9 @@ def get_logintoken(self) -> str: self.log_error(r11.status_code, "logintoken") # --- if not str(r11.status_code).startswith("2"): - printe.output(f"<> {botname} {r11.status_code} Server Error: Server Hangup for url: {self.endpoint}") + printe.output( + f"<> {botname} {r11.status_code} Server Error: Server Hangup for url: {self.endpoint}" + ) # --- except Exception as e: exception_err(e) @@ -293,7 +297,9 @@ def _handle_server_error(self, req0, action, params=None): self.log_error(req0.status_code, action, params=params) # --- if not str(req0.status_code).startswith("2"): - printe.output(f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}") + printe.output( + f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}" + ) def raw_request(self, params, files=None, timeout=30): # --- @@ -411,7 +417,9 @@ def get_rest_result(self, url) -> dict: req0 = seasons_by_lang[self.sea_key].request("GET", url, headers=self.headers) # --- if not str(req0.status_code).startswith("2"): - printe.output(f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}") + printe.output( + f"<> {botname} {req0.status_code} Server Error: Server Hangup for url: {self.endpoint}" + ) # --- except Exception as e: exception_err(e) diff --git a/newapi/super/bot_new.py b/newapi/super/bot_new.py index 5bb4eb2..94d7034 100644 --- a/newapi/super/bot_new.py +++ b/newapi/super/bot_new.py @@ -5,23 +5,24 @@ Exception:{'login': {'result': 'Failed', 'reason': 'You have made too many recent login attempts. Please wait 5 minutes before trying again.'}} """ -import sys -import os + import copy -import requests +import logging +import os +import sys from http.cookiejar import MozillaCookieJar +import requests + from ..api_utils import printe from ..api_utils.except_err import exception_err -from .cookies_bot import get_file_name, del_cookies_file - -from .params_help import PARAMS_HELPS -from .Login_db.bot import log_one from ..api_utils.user_agent import default_user_agent -# import mwclient - -# from mwclient.client import Site +from .cookies_bot import del_cookies_file, get_file_name from .mwclient.client import Site +from .params_help import PARAMS_HELPS + +logger = logging.getLogger(__name__) + # cookies = get_cookies(lang, family, username) logins_count = {1: 0} @@ -46,7 +47,7 @@ def __init__(self, lang, family): # self._start_() def log_error(self, result, action, params=None) -> None: - log_one(site=f"{self.lang}.{self.family}.org", user=self.username, result=result, action=action, params=params) + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def _start_(self, username, password): self.username = username @@ -78,10 +79,14 @@ def __initialize_site(self): self.domain = f"{self.lang}.{self.family}.org" if "dopost" in sys.argv: - self.site_mwclient = Site(self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login) + self.site_mwclient = Site( + self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login + ) else: try: - self.site_mwclient = Site(self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login) + self.site_mwclient = Site( + self.domain, clients_useragent=self.user_agent, pool=self.connection, force_login=self.force_login + ) except Exception as e: printe.output(f"Could not connect to ({self.domain}): %s" % e) return False @@ -161,6 +166,7 @@ def do_request(self, params=None, method="POST"): # ----- # ----- + class LOGIN_HELPS(MwClientSite, PARAMS_HELPS): def __init__(self) -> None: # --- @@ -188,7 +194,7 @@ def add_User_tables(self, family, table, lang="") -> None: if table["username"].find("bot") == -1 and family == "wikipedia": print(f"add_User_tables: {family=}, {table['username']=}") # --- - if family != "" and table['username'] != "" and table['password'] != "": + if family != "" and table["username"] != "" and table["password"] != "": # --- if self.family == family or (langx == "ar" and self.family.startswith("wik")): # wiktionary self.user_table_done = True diff --git a/newapi/super/cookies_bot.py b/newapi/super/cookies_bot.py index 5e6abfc..1467597 100644 --- a/newapi/super/cookies_bot.py +++ b/newapi/super/cookies_bot.py @@ -4,12 +4,14 @@ # cookies = get_cookies(lang, family, username) """ -import sys + import os import stat +import sys +from datetime import datetime, timedelta from functools import lru_cache from pathlib import Path -from datetime import datetime, timedelta + from ..api_utils import printe statgroup = stat.S_IRWXU | stat.S_IRWXG @@ -46,7 +48,7 @@ def get_file_name(lang, family, username) -> Path: # --- if "nocookies" in sys.argv: randome = os.urandom(8).hex() - return ta_dir /f"{randome}.txt" + return ta_dir / f"{randome}.txt" # --- lang = lang.lower() family = family.lower() diff --git a/newapi/super/handel_errors.py b/newapi/super/handel_errors.py index 1916be0..f060eb8 100644 --- a/newapi/super/handel_errors.py +++ b/newapi/super/handel_errors.py @@ -2,7 +2,9 @@ from .super.handel_errors import HANDEL_ERRORS """ + import sys + # from newapi import printe from ..api_utils import printe @@ -55,7 +57,12 @@ def handel_err(self, error: dict, function: str = "", params: dict = None, do_er abusefilter = error.get("abusefilter", "") description = abusefilter.get("description", "") printe.output(f"<> ** abusefilter-disallowed: {description} ") - if description in ["تأخير البوتات 3 ساعات", "تأخير البوتات 3 ساعات- 3 من 3", "تأخير البوتات 3 ساعات- 1 من 3", "تأخير البوتات 3 ساعات- 2 من 3"]: + if description in [ + "تأخير البوتات 3 ساعات", + "تأخير البوتات 3 ساعات- 3 من 3", + "تأخير البوتات 3 ساعات- 1 من 3", + "تأخير البوتات 3 ساعات- 2 من 3", + ]: return False return description # --- @@ -77,8 +84,8 @@ def handel_err(self, error: dict, function: str = "", params: dict = None, do_er return False # --- if do_error: - params['data'] = {} - params['text'] = {} + params["data"] = {} + params["text"] = {} printe.error(f"<>{function} ERROR: <>info: {err_info}, {params=}") # --- if "raise" in sys.argv: diff --git a/newapi/super/login_wrap.py b/newapi/super/login_wrap.py index 317b9e0..c93a891 100644 --- a/newapi/super/login_wrap.py +++ b/newapi/super/login_wrap.py @@ -8,11 +8,13 @@ # bots_login_cache.update(catbots_login2) """ + from ..api_utils import printe from .super_login import Login hases = {} + def LoginWrap(sitecode, family, bots_login_cache, User_tables): # --- cache_key = (sitecode, family) # Consider adding relevant kwargs to key @@ -30,7 +32,10 @@ def LoginWrap(sitecode, family, bots_login_cache, User_tables): hases[cache_key] += 1 # --- if hases[cache_key] % 100 == 0: - printe.output(f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {hases[cache_key]}", p=True) + printe.output( + f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {hases[cache_key]}", + p=True, + ) else: login_bot = Login(sitecode, family=family) # --- diff --git a/newapi/super/login_wrap_new.py b/newapi/super/login_wrap_new.py index 9134181..33fea41 100644 --- a/newapi/super/login_wrap_new.py +++ b/newapi/super/login_wrap_new.py @@ -7,7 +7,9 @@ # login_bot = LoginWrap(sitecode, family, User_tables) """ + from functools import lru_cache + from ..api_utils import printe from .super_login import Login @@ -45,7 +47,9 @@ def LoginWrap(sitecode, family, bots_login_cache, User_tables): # --- cache_info = _create_login_bot.cache_info() if cache_info.hits > 0 and cache_info.hits % 100 == 0: - printe.output(f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {cache_info.hits}", p=True) + printe.output( + f"### <> LoginWrap has bot for ({sitecode}.{family}.org|{username}) count: {cache_info.hits}", p=True + ) # --- # Return bots_login_cache for backward compatibility return login_bot, bots_login_cache diff --git a/newapi/super/mwclient/__init__.py b/newapi/super/mwclient/__init__.py index f16d785..ee90946 100644 --- a/newapi/super/mwclient/__init__.py +++ b/newapi/super/mwclient/__init__.py @@ -1,34 +1,35 @@ """ - Copyright (c) 2006-2011 Bryan Tong Minh +Copyright (c) 2006-2011 Bryan Tong Minh - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. """ -from .errors import * # noqa: F401, F403 -from .client import Site, __version__ # noqa: F401 import logging import warnings +from .client import Site, __version__ # noqa: F401 +from .errors import * # noqa: F401, F403 + # Show DeprecationWarning -warnings.simplefilter('always', DeprecationWarning) +warnings.simplefilter("always", DeprecationWarning) logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/newapi/super/mwclient/client.py b/newapi/super/mwclient/client.py index c069443..9d0db6e 100644 --- a/newapi/super/mwclient/client.py +++ b/newapi/super/mwclient/client.py @@ -1,25 +1,21 @@ -import warnings +import json import logging - +import warnings from collections import OrderedDict -import json import requests -from requests.auth import HTTPBasicAuth, AuthBase +from requests.auth import AuthBase, HTTPBasicAuth from requests_oauthlib import OAuth1 +from . import errors, listing -from . import errors -from . import listing # from .sleep import Sleepers -from .util import parse_timestamp, read_in_chunks, handle_limit -from ..Login_db.bot import log_one +from .util import handle_limit, parse_timestamp, read_in_chunks -__version__ = '0.11.0' +__version__ = "0.11.0" -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) -USER_AGENT = 'mwclient/{} ({})'.format(__version__, - 'https://github.com/mwclient/mwclient') +USER_AGENT = "mwclient/{} ({})".format(__version__, "https://github.com/mwclient/mwclient") class Site: @@ -88,14 +84,34 @@ class Site: (where e is the exception object) and will be one of the API:Login errors. The most common error code is "Failed", indicating a wrong username or password. """ + api_limit = 500 - def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, - max_retries=25, wait_callback=lambda *x: None, clients_useragent=None, - max_lag=3, compress=True, force_login=True, do_init=True, httpauth=None, - connection_options=None, consumer_token=None, consumer_secret=None, - access_token=None, access_secret=None, client_certificate=None, - custom_headers=None, scheme='https', reqs=None): + def __init__( + self, + host, + path="/w/", + ext=".php", + pool=None, + retry_timeout=30, + max_retries=25, + wait_callback=lambda *x: None, + clients_useragent=None, + max_lag=3, + compress=True, + force_login=True, + do_init=True, + httpauth=None, + connection_options=None, + consumer_token=None, + consumer_secret=None, + access_token=None, + access_secret=None, + client_certificate=None, + custom_headers=None, + scheme="https", + reqs=None, + ): # Setup member variables self.host = host self.path = path @@ -106,19 +122,14 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, self.max_lag = str(max_lag) self.force_login = force_login if reqs and connection_options: - print(ValueError( - "reqs is a deprecated alias of connection_options. Do not specify both." - )) + print(ValueError("reqs is a deprecated alias of connection_options. Do not specify both.")) if reqs: - warnings.warn( - "reqs is deprecated in mwclient 1.0.0. Use connection_options instead", - DeprecationWarning - ) + warnings.warn("reqs is deprecated in mwclient 1.0.0. Use connection_options instead", DeprecationWarning) connection_options = reqs self.requests = connection_options or {} self.scheme = scheme - if 'timeout' not in self.requests: - self.requests['timeout'] = 30 # seconds + if "timeout" not in self.requests: + self.requests["timeout"] = 30 # seconds if consumer_token is not None: auth = OAuth1(consumer_token, consumer_secret, access_token, access_secret) @@ -126,24 +137,22 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, # workaround weird requests default to encode as latin-1 # https://github.com/mwclient/mwclient/issues/315 # https://github.com/psf/requests/issues/4564 - httpauth = [ - it.encode("utf-8") if isinstance(it, str) else it for it in httpauth - ] + httpauth = [it.encode("utf-8") if isinstance(it, str) else it for it in httpauth] auth = HTTPBasicAuth(*httpauth) elif httpauth is None or isinstance(httpauth, (AuthBase,)): auth = httpauth else: # FIXME: Raise a specific exception instead of a generic RuntimeError. - print(RuntimeError('Authentication is not a tuple or an instance of AuthBase')) + print(RuntimeError("Authentication is not a tuple or an instance of AuthBase")) # self.sleepers = Sleepers(max_retries, retry_timeout, wait_callback) # Site properties - self.blocked = False # Whether current user is blocked + self.blocked = False # Whether current user is blocked self.hasmsg = False # Whether current user has new messages - self.groups = [] # Groups current user belongs to - self.rights = [] # Rights current user has - self.tokens = {} # Edit tokens of the current user + self.groups = [] # Groups current user belongs to + self.rights = [] # Rights current user has + self.tokens = {} # Edit tokens of the current user self.version = None self.namespaces = self.default_namespaces @@ -157,10 +166,10 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, # Set User-Agent header field if clients_useragent: - ua = clients_useragent + ' ' + USER_AGENT + ua = clients_useragent + " " + USER_AGENT else: ua = USER_AGENT - self.connection.headers['User-Agent'] = ua + self.connection.headers["User-Agent"] = ua if custom_headers: self.connection.headers.update(custom_headers) @@ -187,11 +196,11 @@ def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, try: self.site_init() except errors.APIError as e: - if e.args[0] == 'mwoauth-invalid-authorization': + if e.args[0] == "mwoauth-invalid-authorization": print(errors.OAuthAuthorizationError(self, e.code, e.info)) # Private wiki, do init after login - if e.args[0] not in {'unknown_action', 'readapidenied'}: + if e.args[0] not in {"unknown_action", "readapidenied"}: # raise print("raise") @@ -201,39 +210,38 @@ def site_init(self): `do_init=False` constructor argument.""" if self.initialized: - info = self.get('query', meta='userinfo', uiprop='groups|rights') - userinfo = info['query']['userinfo'] - self.username = userinfo['name'] - self.groups = userinfo.get('groups', []) - self.rights = userinfo.get('rights', []) + info = self.get("query", meta="userinfo", uiprop="groups|rights") + userinfo = info["query"]["userinfo"] + self.username = userinfo["name"] + self.groups = userinfo.get("groups", []) + self.rights = userinfo.get("rights", []) self.tokens = {} return - meta = self.get('query', meta='siteinfo|userinfo', - siprop='general|namespaces', uiprop='groups|rights', - retry_on_error=False) + meta = self.get( + "query", meta="siteinfo|userinfo", siprop="general|namespaces", uiprop="groups|rights", retry_on_error=False + ) # Extract site info - self.site = meta['query']['general'] + self.site = meta["query"]["general"] self.namespaces = { - namespace['id']: namespace.get('*', '') - for namespace in meta['query']['namespaces'].values() + namespace["id"]: namespace.get("*", "") for namespace in meta["query"]["namespaces"].values() } - self.version = self.version_tuple_from_generator(self.site['generator']) + self.version = self.version_tuple_from_generator(self.site["generator"]) # Require MediaWiki version >= 1.16 self.require(1, 16) # User info - userinfo = meta['query']['userinfo'] - self.username = userinfo['name'] - self.groups = userinfo.get('groups', []) - self.rights = userinfo.get('rights', []) + userinfo = meta["query"]["userinfo"] + self.username = userinfo["name"] + self.groups = userinfo.get("groups", []) + self.rights = userinfo.get("rights", []) self.initialized = True @staticmethod - def version_tuple_from_generator(string, prefix='MediaWiki '): + def version_tuple_from_generator(string, prefix="MediaWiki "): """Return a version tuple from a MediaWiki Generator string. Example: @@ -248,9 +256,9 @@ def version_tuple_from_generator(string, prefix='MediaWiki '): A tuple containing the individual elements of the given version number. """ if not string.startswith(prefix): - print(errors.MediaWikiVersionError('Unknown generator {}'.format(string))) + print(errors.MediaWikiVersionError("Unknown generator {}".format(string))) - version = string[len(prefix):].split('.') + version = string[len(prefix) :].split(".") def split_num(s): """Split the string on the first non-digit character. @@ -261,28 +269,43 @@ def split_num(s): """ i = 0 while i < len(s): - if s[i] < '0' or s[i] > '9': + if s[i] < "0" or s[i] > "9": break i += 1 if s[i:]: - return (int(s[:i]), s[i:], ) + return ( + int(s[:i]), + s[i:], + ) else: - return (int(s[:i]), ) + return (int(s[:i]),) version_tuple = sum((split_num(s) for s in version), ()) if len(version_tuple) < 2: - print(errors.MediaWikiVersionError('Unknown MediaWiki {}' - .format('.'.join(version)))) + print(errors.MediaWikiVersionError("Unknown MediaWiki {}".format(".".join(version)))) return version_tuple default_namespaces = { - 0: '', 1: 'Talk', 2: 'User', 3: 'User talk', 4: 'Project', - 5: 'Project talk', 6: 'Image', 7: 'Image talk', 8: 'MediaWiki', - 9: 'MediaWiki talk', 10: 'Template', 11: 'Template talk', 12: 'Help', - 13: 'Help talk', 14: 'Category', 15: 'Category talk', - -1: 'Special', -2: 'Media' + 0: "", + 1: "Talk", + 2: "User", + 3: "User talk", + 4: "Project", + 5: "Project talk", + 6: "Image", + 7: "Image talk", + 8: "MediaWiki", + 9: "MediaWiki talk", + 10: "Template", + 11: "Template talk", + 12: "Help", + 13: "Help talk", + 14: "Category", + 15: "Category talk", + -1: "Special", + -2: "Media", } def __repr__(self): @@ -300,7 +323,7 @@ def get(self, action, *args, **kwargs): Returns: The raw response from the API call, as a dictionary. """ - return self.api(action, 'GET', *args, **kwargs) + return self.api(action, "GET", *args, **kwargs) def post(self, action, *args, **kwargs): """Perform a generic API call using POST. @@ -314,9 +337,9 @@ def post(self, action, *args, **kwargs): Returns: The raw response from the API call, as a dictionary. """ - return self.api(action, 'POST', *args, **kwargs) + return self.api(action, "POST", *args, **kwargs) - def api(self, action, http_method='POST', *args, **kwargs): + def api(self, action, http_method="POST", *args, **kwargs): """Perform a generic API call and handle errors. All arguments will be passed on. @@ -343,17 +366,17 @@ def api(self, action, http_method='POST', *args, **kwargs): """ kwargs.update(args) - if action == 'query' and 'continue' not in kwargs: - kwargs['continue'] = '' - if action == 'query': - if 'meta' in kwargs: - kwargs['meta'] += '|userinfo' + if action == "query" and "continue" not in kwargs: + kwargs["continue"] = "" + if action == "query": + if "meta" in kwargs: + kwargs["meta"] += "|userinfo" else: - kwargs['meta'] = 'userinfo' - if 'uiprop' in kwargs: - kwargs['uiprop'] += '|blockinfo|hasmsg' + kwargs["meta"] = "userinfo" + if "uiprop" in kwargs: + kwargs["uiprop"] += "|blockinfo|hasmsg" else: - kwargs['uiprop'] = 'blockinfo|hasmsg' + kwargs["uiprop"] = "blockinfo|hasmsg" # sleeper = self.sleepers.make() @@ -362,11 +385,11 @@ def api(self, action, http_method='POST', *args, **kwargs): if not info: info = {} # if self.handle_api_result(info, sleeper=sleeper): - self.handle_api_result(info)#, sleeper=sleeper + self.handle_api_result(info) # , sleeper=sleeper return info def log_error(self, result, action, params=None) -> None: - log_one(site=self.host, user=self.username, result=result, action=action, params=params) + logger.error(f"Error occurred: {result}, Action: {action}, Params: {params}") def handle_api_result(self, info, kwargs=None, sleeper=None): """Checks the given API response, raising an appropriate exception or sleeping if @@ -386,59 +409,50 @@ def handle_api_result(self, info, kwargs=None, sleeper=None): # if sleeper is None: sleeper = self.sleepers.make() try: - userinfo = info['query']['userinfo'] + userinfo = info["query"]["userinfo"] except KeyError: userinfo = () - if 'blockedby' in userinfo: - self.blocked = (userinfo['blockedby'], userinfo.get('blockreason', '')) + if "blockedby" in userinfo: + self.blocked = (userinfo["blockedby"], userinfo.get("blockreason", "")) else: self.blocked = False - self.hasmsg = 'messages' in userinfo - self.logged_in = 'anon' not in userinfo and 'temp' not in userinfo - if 'warnings' in info: - for module, warning in info['warnings'].items(): - if '*' in warning: - log.warning(warning['*']) - - if 'error' in info: - if info['error'].get('code') in {'internal_api_error_DBConnectionError', - 'internal_api_error_DBQueryError'}: + self.hasmsg = "messages" in userinfo + self.logged_in = "anon" not in userinfo and "temp" not in userinfo + if "warnings" in info: + for module, warning in info["warnings"].items(): + if "*" in warning: + logger.warning(warning["*"]) + + if "error" in info: + if info["error"].get("code") in {"internal_api_error_DBConnectionError", "internal_api_error_DBQueryError"}: # sleeper.sleep() return False # cope with https://phabricator.wikimedia.org/T106066 - if ( - info['error'].get('code') == 'mwoauth-invalid-authorization' - and 'Nonce already used' in info['error'].get('info') - ): - log.warning('Retrying due to nonce error, see' - 'https://phabricator.wikimedia.org/T106066') + if info["error"].get("code") == "mwoauth-invalid-authorization" and "Nonce already used" in info[ + "error" + ].get("info"): + logger.warning("Retrying due to nonce error, see" "https://phabricator.wikimedia.org/T106066") # sleeper.sleep() return False - if 'query' in info['error']: + if "query" in info["error"]: # Semantic Mediawiki does not follow the standard error format - print(errors.APIError(None, info['error']['query'], kwargs)) + print(errors.APIError(None, info["error"]["query"], kwargs)) - if '*' in info['error']: - print(errors.APIError(info['error']['code'], - info['error']['info'], info['error']['*'])) - print(errors.APIError(info['error']['code'], - info['error']['info'], kwargs)) + if "*" in info["error"]: + print(errors.APIError(info["error"]["code"], info["error"]["info"], info["error"]["*"])) + print(errors.APIError(info["error"]["code"], info["error"]["info"], kwargs)) return True @staticmethod def _query_string(*args, **kwargs): kwargs.update(args) - qs1 = [ - (k, v) for k, v in kwargs.items() if k not in {'wpEditToken', 'token'} - ] - qs2 = [ - (k, v) for k, v in kwargs.items() if k in {'wpEditToken', 'token'} - ] + qs1 = [(k, v) for k, v in kwargs.items() if k not in {"wpEditToken", "token"}] + qs2 = [(k, v) for k, v in kwargs.items() if k in {"wpEditToken", "token"}] return OrderedDict(qs1 + qs2) - def raw_call(self, script, data, files=None, retry_on_error=True, http_method='POST'): + def raw_call(self, script, data, files=None, retry_on_error=True, http_method="POST"): """ Perform a generic request and return the raw text. @@ -471,40 +485,39 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method='P """ headers = {} if self.compress: - headers['Accept-Encoding'] = 'gzip' + headers["Accept-Encoding"] = "gzip" # sleeper = self.sleepers.make((script, data)) scheme = self.scheme host = self.host if isinstance(host, (list, tuple)): warnings.warn( - 'Specifying host as a tuple is deprecated as of mwclient 0.10.1. ' - + 'Please use the new scheme argument instead.', - DeprecationWarning + "Specifying host as a tuple is deprecated as of mwclient 0.10.1. " + + "Please use the new scheme argument instead.", + DeprecationWarning, ) scheme, host = host - url = '{scheme}://{host}{path}{script}{ext}'.format(scheme=scheme, host=host, - path=self.path, script=script, - ext=self.ext) + url = "{scheme}://{host}{path}{script}{ext}".format( + scheme=scheme, host=host, path=self.path, script=script, ext=self.ext + ) # while True: toraise = None wait_time = 0 - args = {'files': files, 'headers': headers} + args = {"files": files, "headers": headers} for k, v in self.requests.items(): args[k] = v - if http_method == 'GET': - args['params'] = data + if http_method == "GET": + args["params"] = data else: - args['data'] = data - maxlag = data.get('maxlag', self.max_lag) + args["data"] = data + maxlag = data.get("maxlag", self.max_lag) try: stream = self.connection.request(http_method, url, **args) - if stream.headers.get('x-database-lag'): - wait_time = int(stream.headers.get('retry-after')) - log.warning('Database lag exceeds max lag. ' - f'Waiting for {wait_time} seconds, maxlag:{maxlag}') + if stream.headers.get("x-database-lag"): + wait_time = int(stream.headers.get("retry-after")) + logger.warning("Database lag exceeds max lag. " f"Waiting for {wait_time} seconds, maxlag:{maxlag}") # fall through to the sleep elif stream.status_code == 200: return stream.text @@ -513,25 +526,22 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method='P else: if not retry_on_error: stream.raise_for_status() - log.warning('Received {status} response: {text}. ' - 'Retrying in a moment.' - .format(status=stream.status_code, - text=stream.text)) + logger.warning( + "Received {status} response: {text}. " + "Retrying in a moment.".format(status=stream.status_code, text=stream.text) + ) toraise = "stream" # fall through to the sleep return stream.text - except ( - requests.exceptions.ConnectionError, - requests.exceptions.Timeout - ) as err: + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as err: # In the event of a network problem # (e.g. DNS failure, refused connection, etc), # Requests will raise a ConnectionError exception. if not retry_on_error: print("raise") print(err) - log.warning('Connection error. Retrying in a moment.') + logger.warning("Connection error. Retrying in a moment.") toraise = err # proceed to the sleep @@ -547,7 +557,7 @@ def raw_call(self, script, data, files=None, retry_on_error=True, http_method='P else: print("raise") - def raw_api(self, action, http_method='POST', retry_on_error=True, *args, **kwargs): + def raw_api(self, action, http_method="POST", retry_on_error=True, *args, **kwargs): """Send a call to the API. Args: @@ -572,11 +582,10 @@ def raw_api(self, action, http_method='POST', retry_on_error=True, *args, **kwar performing the API request. requests.exceptions.Timeout: The API request timed out. """ - kwargs['action'] = action - kwargs['format'] = 'json' + kwargs["action"] = action + kwargs["format"] = "json" data = self._query_string(*args, **kwargs) - res = self.raw_call('api', data, retry_on_error=retry_on_error, - http_method=http_method) + res = self.raw_call("api", data, retry_on_error=retry_on_error, http_method=http_method) try: # data = json.loads(res, object_pairs_hook=OrderedDict) data2 = json.loads(res) @@ -588,11 +597,11 @@ def raw_api(self, action, http_method='POST', retry_on_error=True, *args, **kwar except ValueError: self.log_error("ValueError", action, params=data) # --- - if res.startswith('MediaWiki API is not enabled for this site.'): + if res.startswith("MediaWiki API is not enabled for this site."): print(errors.APIDisabledError) print(errors.InvalidResponse(res)) - def raw_index(self, action, http_method='POST', *args, **kwargs): + def raw_index(self, action, http_method="POST", *args, **kwargs): """Sends a call to index.php rather than the API. Args: @@ -614,11 +623,11 @@ def raw_index(self, action, http_method='POST', *args, **kwargs): performing the API request. requests.exceptions.Timeout: The API request timed out. """ - kwargs['action'] = action + kwargs["action"] = action data = self._query_string(*args, **kwargs) - if not data.get('maxlag'): - data['maxlag'] = self.max_lag - return self.raw_call('index', data, http_method=http_method) + if not data.get("maxlag"): + data["maxlag"] = self.max_lag + return self.raw_call("index", data, http_method=http_method) def require(self, major, minor, revision=None, raise_error=True): """Check whether the current wiki matches the required version. @@ -650,17 +659,19 @@ def require(self, major, minor, revision=None, raise_error=True): if raise_error is None: return # FIXME: Replace this with a specific error - print(RuntimeError('Site %s has not yet been initialized' % repr(self))) + print(RuntimeError("Site %s has not yet been initialized" % repr(self))) if revision is None: if self.version[:2] >= (major, minor): return True elif raise_error: - print(errors.MediaWikiVersionError( - 'Requires version {required[0]}.{required[1]}, ' - 'current version is {current[0]}.{current[1]}' - .format(required=(major, minor), - current=(self.version[:2]))) + print( + errors.MediaWikiVersionError( + "Requires version {required[0]}.{required[1]}, " + "current version is {current[0]}.{current[1]}".format( + required=(major, minor), current=(self.version[:2]) + ) + ) ) else: return False @@ -691,13 +702,12 @@ def email(self, user, text, subject, cc=False): EmailError (mwclient.errors.EmailError): Other email errors """ - token = self.get_token('email') + token = self.get_token("email") try: - info = self.post('emailuser', target=user, subject=subject, - text=text, ccme=cc, token=token) + info = self.post("emailuser", target=user, subject=subject, text=text, ccme=cc, token=token) except errors.APIError as e: - if e.args[0] == 'noemail': + if e.args[0] == "noemail": print(errors.NoSpecifiedEmail(user, e.args[1])) print(errors.EmailError(*e)) @@ -749,40 +759,35 @@ def login(self, username=None, password=None, cookies=None, domain=None): if self.credentials: # sleeper = self.sleepers.make() - kwargs = { - 'lgname': self.credentials[0], - 'lgpassword': self.credentials[1] - } + kwargs = {"lgname": self.credentials[0], "lgpassword": self.credentials[1]} if self.credentials[2]: - kwargs['lgdomain'] = self.credentials[2] + kwargs["lgdomain"] = self.credentials[2] # Try to login using the scheme for MW 1.27+. If the wiki is read protected, # it is not possible to get the wiki version upfront using the API, so we just # have to try. If the attempt fails, we try the old method. try: - kwargs['lgtoken'] = self.get_token('login') + kwargs["lgtoken"] = self.get_token("login") except (errors.APIError, KeyError): - log.debug('Failed to get login token, MediaWiki is older than 1.27.') - + logger.debug("Failed to get login token, MediaWiki is older than 1.27.") # while True: - login = self.post('login', **kwargs) + login = self.post("login", **kwargs) - if login['login']['result'] == 'Success': - login_result = 'Success' + if login["login"]["result"] == "Success": + login_result = "Success" # break - elif login['login']['result'] == 'NeedToken': - login_result = 'NeedToken' - kwargs['lgtoken'] = login['login']['token'] - elif login['login']['result'] == 'Throttled': - login_result = 'Throttled' - so = int(login['login'].get('wait', 5)) + elif login["login"]["result"] == "NeedToken": + login_result = "NeedToken" + kwargs["lgtoken"] = login["login"]["token"] + elif login["login"]["result"] == "Throttled": + login_result = "Throttled" + so = int(login["login"].get("wait", 5)) # sleeper.sleep(so) print(f"so: {so}") else: - login_result = login['login']['result'] - print(errors.LoginError(self, login['login']['result'], - login['login']['reason'])) + login_result = login["login"]["result"] + print(errors.LoginError(self, login["login"]["result"], login["login"]["reason"])) self.site_init() @@ -836,26 +841,25 @@ def clientlogin(self, cookies=None, **kwargs): # Try to login using the scheme for MW 1.27+. If the wiki is read protected, # it is not possible to get the wiki version upfront using the API, so we just # have to try. If the attempt fails, we try the old method. - if 'logintoken' not in kwargs: + if "logintoken" not in kwargs: try: - kwargs['logintoken'] = self.get_token('login') + kwargs["logintoken"] = self.get_token("login") except (errors.APIError, KeyError): - log.debug('Failed to get login token, MediaWiki is older than 1.27.') + logger.debug("Failed to get login token, MediaWiki is older than 1.27.") - if 'logincontinue' not in kwargs and 'loginreturnurl' not in kwargs: + if "logincontinue" not in kwargs and "loginreturnurl" not in kwargs: # should be great if API didn't require this... - kwargs['loginreturnurl'] = '%s://%s' % (self.scheme, self.host) + kwargs["loginreturnurl"] = "%s://%s" % (self.scheme, self.host) # while True: - login = self.post('clientlogin', **kwargs) - status = login['clientlogin'].get('status') - if status == 'PASS': + login = self.post("clientlogin", **kwargs) + status = login["clientlogin"].get("status") + if status == "PASS": return True - elif status in ('UI', 'REDIRECT'): - return login['clientlogin'] + elif status in ("UI", "REDIRECT"): + return login["clientlogin"] else: - print(errors.LoginError(self, status, - login['clientlogin'].get('message'))) + print(errors.LoginError(self, status, login["clientlogin"].get("message"))) def get_token(self, type, force=False, title=None): """Request a MediaWiki access token of the given `type`. @@ -876,41 +880,49 @@ def get_token(self, type, force=False, title=None): if self.version is None or self.version[:2] >= (1, 24): # The 'csrf' (cross-site request forgery) token introduced in 1.24 replaces # the majority of older tokens, like edittoken and movetoken. - if type not in {'watch', 'patrol', 'rollback', 'userrights', 'login'}: - type = 'csrf' + if type not in {"watch", "patrol", "rollback", "userrights", "login"}: + type = "csrf" if type not in self.tokens: - self.tokens[type] = '0' + self.tokens[type] = "0" - if self.tokens.get(type, '0') == '0' or force: + if self.tokens.get(type, "0") == "0" or force: if self.version is None or self.version[:2] >= (1, 24): # We use raw_api() rather than api() because api() is adding "userinfo" # to the query and this raises a readapideniederror if the wiki is read # protected, and we're trying to fetch a login token. - info = self.raw_api('query', 'GET', meta='tokens', type=type) + info = self.raw_api("query", "GET", meta="tokens", type=type) self.handle_api_result(info) # Note that for read protected wikis, we don't know the version when # fetching the login token. If it's < 1.27, the request below will # raise a KeyError that we should catch. - self.tokens[type] = info['query']['tokens']['%stoken' % type] + self.tokens[type] = info["query"]["tokens"]["%stoken" % type] else: if title is None: # Some dummy title was needed to get a token prior to 1.24 - title = 'Test' - info = self.post('query', titles=title, - prop='info', intoken=type) - for i in info['query']['pages'].values(): - if i['title'] == title: - self.tokens[type] = i['%stoken' % type] + title = "Test" + info = self.post("query", titles=title, prop="info", intoken=type) + for i in info["query"]["pages"].values(): + if i["title"] == title: + self.tokens[type] = i["%stoken" % type] return self.tokens[type] - def upload(self, file=None, filename=None, description='', ignore=False, - file_size=None, url=None, filekey=None, comment=None): + def upload( + self, + file=None, + filename=None, + description="", + ignore=False, + file_size=None, + url=None, + filekey=None, + comment=None, + ): """Upload a file to the site. Note that one of `file`, `filekey` and `url` must be specified, but not @@ -945,21 +957,16 @@ def upload(self, file=None, filename=None, description='', ignore=False, if file_size is not None: # Note that DeprecationWarning is hidden by default since Python 2.7 - warnings.warn( - 'file_size is deprecated since mwclient 0.7', - DeprecationWarning - ) + warnings.warn("file_size is deprecated since mwclient 0.7", DeprecationWarning) if filename is None: - print(TypeError('filename must be specified')) + print(TypeError("filename must be specified")) if len([x for x in [file, filekey, url] if x is not None]) != 1: - print(TypeError( - "exactly one of 'file', 'filekey' and 'url' must be specified" - )) + print(TypeError("exactly one of 'file', 'filekey' and 'url' must be specified")) image = self.Images[filename] - if not image.can('upload'): + if not image.can("upload"): print(errors.InsufficientPermission(filename)) if comment is None: @@ -970,8 +977,8 @@ def upload(self, file=None, filename=None, description='', ignore=False, text = description if file is not None: - if not hasattr(file, 'read'): - file = open(file, 'rb') + if not hasattr(file, "read"): + file = open(file, "rb") content_size = file.seek(0, 2) file.seek(0) @@ -980,25 +987,25 @@ def upload(self, file=None, filename=None, description='', ignore=False, return self.chunk_upload(file, filename, ignore, comment, text) predata = { - 'action': 'upload', - 'format': 'json', - 'filename': filename, - 'comment': comment, - 'text': text, - 'token': image.get_token('edit'), + "action": "upload", + "format": "json", + "filename": filename, + "comment": comment, + "text": text, + "token": image.get_token("edit"), } if ignore: - predata['ignorewarnings'] = 'true' + predata["ignorewarnings"] = "true" if url: - predata['url'] = url + predata["url"] = url # sessionkey was renamed to filekey in MediaWiki 1.18 # https://phabricator.wikimedia.org/rMW5f13517e36b45342f228f3de4298bb0fe186995d if self.version[:2] < (1, 18): - predata['sessionkey'] = filekey + predata["sessionkey"] = filekey else: - predata['filekey'] = filekey + predata["filekey"] = filekey postdata = predata files = None @@ -1009,25 +1016,27 @@ def upload(self, file=None, filename=None, description='', ignore=False, # Since the filename in Content-Disposition is not interpreted, # we can send some ascii-only dummy name rather than the real # filename, which might contain non-ascii. - files = {'file': ('fake-filename', file)} + files = {"file": ("fake-filename", file)} # sleeper = self.sleepers.make() # while True: - data = self.raw_call('api', postdata, files) + data = self.raw_call("api", postdata, files) info = json.loads(data) if not info: info = {} - if self.handle_api_result(info, kwargs=predata - #, sleeper=sleeper - ): - response = info.get('upload', {}) + if self.handle_api_result( + info, + kwargs=predata, + # , sleeper=sleeper + ): + response = info.get("upload", {}) # Workaround for https://github.com/mwclient/mwclient/issues/211 # ---------------------------------------------------------------- # Raise an error if the file already exists. This is necessary because # MediaWiki returns a warning, not an error, leading to silent failure. # The user must explicitly set ignore=True (ignorewarnings=True) to # overwrite an existing file. - if ignore is False and 'exists' in response.get('warnings', {}): + if ignore is False and "exists" in response.get("warnings", {}): print(errors.FileExists(filename)) # break @@ -1052,36 +1061,38 @@ def chunk_upload(self, file, filename, ignorewarnings, comment, text): file.seek(0) params = { - 'action': 'upload', - 'format': 'json', - 'stash': 1, - 'offset': 0, - 'filename': filename, - 'filesize': content_size, - 'token': image.get_token('edit'), + "action": "upload", + "format": "json", + "stash": 1, + "offset": 0, + "filename": filename, + "filesize": content_size, + "token": image.get_token("edit"), } if ignorewarnings: - params['ignorewarnings'] = 'true' + params["ignorewarnings"] = "true" # sleeper = self.sleepers.make() offset = 0 for chunk in read_in_chunks(file, self.chunk_size): while True: - data = self.raw_call('api', params, files={'chunk': chunk}) + data = self.raw_call("api", params, files={"chunk": chunk}) info = json.loads(data) - if self.handle_api_result(info, kwargs=params - #, sleeper=sleeper + if self.handle_api_result( + info, + kwargs=params, + # , sleeper=sleeper ): - response = info.get('upload', {}) + response = info.get("upload", {}) break offset += chunk.tell() chunk.close() - log.debug('%s: Uploaded %d of %d bytes', filename, offset, content_size) - params['filekey'] = response['filekey'] - if response['result'] == 'Continue': - params['offset'] = response['offset'] - elif response['result'] == 'Success': + logger.debug("%s: Uploaded %d of %d bytes", filename, offset, content_size) + params["filekey"] = response["filekey"] + if response["result"] == "Continue": + params["offset"] = response["offset"] + elif response["result"] == "Success": file.close() break else: @@ -1091,15 +1102,14 @@ def chunk_upload(self, file, filename, ignorewarnings, comment, text): file.close() return response - del params['action'] - del params['stash'] - del params['offset'] - params['comment'] = comment - params['text'] = text - return self.post('upload', **params) + del params["action"] + del params["stash"] + del params["offset"] + params["comment"] = comment + params["text"] = text + return self.post("upload", **params) - def parse(self, text=None, title=None, page=None, prop=None, - redirects=False, mobileformat=False): + def parse(self, text=None, title=None, page=None, prop=None, redirects=False, mobileformat=False): """Parses the given content and returns parser output. Args: @@ -1119,19 +1129,19 @@ def parse(self, text=None, title=None, page=None, prop=None, """ kwargs = {} if text is not None: - kwargs['text'] = text + kwargs["text"] = text if title is not None: - kwargs['title'] = title + kwargs["title"] = title if page is not None: - kwargs['page'] = page + kwargs["page"] = page if prop is not None: - kwargs['prop'] = prop + kwargs["prop"] = prop if redirects: - kwargs['redirects'] = '1' + kwargs["redirects"] = "1" if mobileformat: - kwargs['mobileformat'] = '1' - result = self.post('parse', **kwargs) - return result['parse'] + kwargs["mobileformat"] = "1" + result = self.post("parse", **kwargs) + return result["parse"] # def block(self): TODO? # def unblock: TODO? @@ -1167,101 +1177,192 @@ def patrol(self, rcid=None, revid=None, tags=None): - ``tags`` requires at least MediaWiki 1.27. """ if self.require(1, 17, raise_error=False): - token = self.get_token('patrol') + token = self.get_token("patrol") else: # For MediaWiki versions earlier than 1.17, the patrol token is the same the # edit token. - token = self.get_token('edit') + token = self.get_token("edit") - result = self.post('patrol', rcid=rcid, revid=revid, tags=tags, token=token) - return result['patrol'] + result = self.post("patrol", rcid=rcid, revid=revid, tags=tags, token=token) + return result["patrol"] # Lists - def allpages(self, start=None, prefix=None, namespace='0', filterredir='all', - minsize=None, maxsize=None, prtype=None, prlevel=None, - limit=None, dir='ascending', filterlanglinks='all', generator=True, - end=None, max_items=None, api_chunk_size=None): + def allpages( + self, + start=None, + prefix=None, + namespace="0", + filterredir="all", + minsize=None, + maxsize=None, + prtype=None, + prlevel=None, + limit=None, + dir="ascending", + filterlanglinks="all", + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all pages on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('ap', generator) - kwargs = dict(listing.List.generate_kwargs( - pfx, ('from', start), ('to', end), prefix=prefix, - minsize=minsize, maxsize=maxsize, prtype=prtype, prlevel=prlevel, - namespace=namespace, filterredir=filterredir, dir=dir, - filterlanglinks=filterlanglinks, - )) - return listing.List.get_list(generator)(self, 'allpages', 'ap', - max_items=max_items, - api_chunk_size=api_chunk_size, - return_values='title', - **kwargs) - - def allimages(self, start=None, prefix=None, minsize=None, maxsize=None, limit=None, - dir='ascending', sha1=None, sha1base36=None, generator=True, end=None, - max_items=None, api_chunk_size=None): + pfx = listing.List.get_prefix("ap", generator) + kwargs = dict( + listing.List.generate_kwargs( + pfx, + ("from", start), + ("to", end), + prefix=prefix, + minsize=minsize, + maxsize=maxsize, + prtype=prtype, + prlevel=prlevel, + namespace=namespace, + filterredir=filterredir, + dir=dir, + filterlanglinks=filterlanglinks, + ) + ) + return listing.List.get_list(generator)( + self, "allpages", "ap", max_items=max_items, api_chunk_size=api_chunk_size, return_values="title", **kwargs + ) + + def allimages( + self, + start=None, + prefix=None, + minsize=None, + maxsize=None, + limit=None, + dir="ascending", + sha1=None, + sha1base36=None, + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all images on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('ai', generator) - kwargs = dict(listing.List.generate_kwargs( - pfx, ('from', start), ('to', end), prefix=prefix, - minsize=minsize, maxsize=maxsize, - dir=dir, sha1=sha1, sha1base36=sha1base36, - )) - return listing.List.get_list(generator)(self, 'allimages', 'ai', - max_items=max_items, - api_chunk_size=api_chunk_size, - return_values='timestamp|url', - **kwargs) - - def alllinks(self, start=None, prefix=None, unique=False, prop='title', - namespace='0', limit=None, generator=True, end=None, max_items=None, - api_chunk_size=None): + pfx = listing.List.get_prefix("ai", generator) + kwargs = dict( + listing.List.generate_kwargs( + pfx, + ("from", start), + ("to", end), + prefix=prefix, + minsize=minsize, + maxsize=maxsize, + dir=dir, + sha1=sha1, + sha1base36=sha1base36, + ) + ) + return listing.List.get_list(generator)( + self, + "allimages", + "ai", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="timestamp|url", + **kwargs, + ) + + def alllinks( + self, + start=None, + prefix=None, + unique=False, + prop="title", + namespace="0", + limit=None, + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve a list of all links on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('al', generator) - kwargs = dict(listing.List.generate_kwargs(pfx, ('from', start), ('to', end), - prefix=prefix, - prop=prop, namespace=namespace)) + pfx = listing.List.get_prefix("al", generator) + kwargs = dict( + listing.List.generate_kwargs( + pfx, ("from", start), ("to", end), prefix=prefix, prop=prop, namespace=namespace + ) + ) if unique: - kwargs[pfx + 'unique'] = '1' - return listing.List.get_list(generator)(self, 'alllinks', 'al', - max_items=max_items, - api_chunk_size=api_chunk_size, - return_values='title', **kwargs) - - def allcategories(self, start=None, prefix=None, dir='ascending', limit=None, - generator=True, end=None, max_items=None, api_chunk_size=None): + kwargs[pfx + "unique"] = "1" + return listing.List.get_list(generator)( + self, "alllinks", "al", max_items=max_items, api_chunk_size=api_chunk_size, return_values="title", **kwargs + ) + + def allcategories( + self, + start=None, + prefix=None, + dir="ascending", + limit=None, + generator=True, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all categories on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - pfx = listing.List.get_prefix('ac', generator) - kwargs = dict(listing.List.generate_kwargs(pfx, ('from', start), ('to', end), - prefix=prefix, dir=dir)) - return listing.List.get_list(generator)(self, 'allcategories', 'ac', - max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def allusers(self, start=None, prefix=None, group=None, prop=None, limit=None, - witheditsonly=False, activeusers=False, rights=None, end=None, - max_items=None, api_chunk_size=None): + pfx = listing.List.get_prefix("ac", generator) + kwargs = dict(listing.List.generate_kwargs(pfx, ("from", start), ("to", end), prefix=prefix, dir=dir)) + return listing.List.get_list(generator)( + self, "allcategories", "ac", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + ) + + def allusers( + self, + start=None, + prefix=None, + group=None, + prop=None, + limit=None, + witheditsonly=False, + activeusers=False, + rights=None, + end=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve all users on the wiki as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('au', ('from', start), ('to', end), - prefix=prefix, - group=group, prop=prop, - rights=rights, - witheditsonly=witheditsonly, - activeusers=activeusers)) - return listing.List(self, 'allusers', 'au', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def blocks(self, start=None, end=None, dir='older', ids=None, users=None, limit=None, - prop='id|user|by|timestamp|expiry|reason|flags', max_items=None, - api_chunk_size=None): + kwargs = dict( + listing.List.generate_kwargs( + "au", + ("from", start), + ("to", end), + prefix=prefix, + group=group, + prop=prop, + rights=rights, + witheditsonly=witheditsonly, + activeusers=activeusers, + ) + ) + return listing.List(self, "allusers", "au", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def blocks( + self, + start=None, + end=None, + dir="older", + ids=None, + users=None, + limit=None, + prop="id|user|by|timestamp|expiry|reason|flags", + max_items=None, + api_chunk_size=None, + ): """Retrieve blocks as a generator. API doc: https://www.mediawiki.org/wiki/API:Blocks @@ -1288,23 +1389,30 @@ def blocks(self, start=None, end=None, dir='older', ids=None, users=None, limit= # TODO: Fix. Fix what? (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('bk', start=start, end=end, dir=dir, - ids=ids, users=users, prop=prop)) - return listing.List(self, 'blocks', 'bk', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def deletedrevisions(self, start=None, end=None, dir='older', namespace=None, - limit=None, prop='user|comment', max_items=None, - api_chunk_size=None): + kwargs = dict( + listing.List.generate_kwargs("bk", start=start, end=end, dir=dir, ids=ids, users=users, prop=prop) + ) + return listing.List(self, "blocks", "bk", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def deletedrevisions( + self, + start=None, + end=None, + dir="older", + namespace=None, + limit=None, + prop="user|comment", + max_items=None, + api_chunk_size=None, + ): # TODO: Fix (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('dr', start=start, end=end, dir=dir, - namespace=namespace, prop=prop)) - return listing.List(self, 'deletedrevs', 'dr', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) + kwargs = dict(listing.List.generate_kwargs("dr", start=start, end=end, dir=dir, namespace=namespace, prop=prop)) + return listing.List(self, "deletedrevs", "dr", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) - def exturlusage(self, query, prop=None, protocol='http', namespace=None, limit=None, - max_items=None, api_chunk_size=None): + def exturlusage( + self, query, prop=None, protocol="http", namespace=None, limit=None, max_items=None, api_chunk_size=None + ): r"""Retrieve the list of pages that link to a particular domain or URL, as a generator. @@ -1327,33 +1435,44 @@ def exturlusage(self, query, prop=None, protocol='http', namespace=None, limit=N """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('eu', query=query, prop=prop, - protocol=protocol, - namespace=namespace)) - return listing.List(self, 'exturlusage', 'eu', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def logevents(self, type=None, prop=None, start=None, end=None, - dir='older', user=None, title=None, limit=None, action=None, - max_items=None, api_chunk_size=None): + kwargs = dict( + listing.List.generate_kwargs("eu", query=query, prop=prop, protocol=protocol, namespace=namespace) + ) + return listing.List(self, "exturlusage", "eu", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def logevents( + self, + type=None, + prop=None, + start=None, + end=None, + dir="older", + user=None, + title=None, + limit=None, + action=None, + max_items=None, + api_chunk_size=None, + ): """Retrieve logevents as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('le', prop=prop, type=type, - start=start, end=end, dir=dir, - user=user, title=title, action=action)) - return listing.List(self, 'logevents', 'le', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def checkuserlog(self, user=None, target=None, limit=None, dir='older', - start=None, end=None, max_items=None, api_chunk_size=10): + kwargs = dict( + listing.List.generate_kwargs( + "le", prop=prop, type=type, start=start, end=end, dir=dir, user=user, title=title, action=action + ) + ) + return listing.List(self, "logevents", "le", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def checkuserlog( + self, user=None, target=None, limit=None, dir="older", start=None, end=None, max_items=None, api_chunk_size=10 + ): """Retrieve checkuserlog items as a generator.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('cul', target=target, start=start, - end=end, dir=dir, user=user)) - return listing.NestedList('entries', self, 'checkuserlog', 'cul', - max_items=max_items, api_chunk_size=api_chunk_size, - **kwargs) + kwargs = dict(listing.List.generate_kwargs("cul", target=target, start=start, end=end, dir=dir, user=user)) + return listing.NestedList( + "entries", self, "checkuserlog", "cul", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + ) # def protectedtitles requires 1.15 def random(self, namespace, limit=None, max_items=None, api_chunk_size=20): @@ -1368,24 +1487,41 @@ def random(self, namespace, limit=None, max_items=None, api_chunk_size=20): """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('rn', namespace=namespace)) - return listing.List(self, 'random', 'rn', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def recentchanges(self, start=None, end=None, dir='older', namespace=None, - prop=None, show=None, limit=None, type=None, toponly=None, - max_items=None, api_chunk_size=None): - """List recent changes to the wiki, à la Special:Recentchanges. - """ + kwargs = dict(listing.List.generate_kwargs("rn", namespace=namespace)) + return listing.List(self, "random", "rn", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def recentchanges( + self, + start=None, + end=None, + dir="older", + namespace=None, + prop=None, + show=None, + limit=None, + type=None, + toponly=None, + max_items=None, + api_chunk_size=None, + ): + """List recent changes to the wiki, à la Special:Recentchanges.""" (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('rc', start=start, end=end, dir=dir, - namespace=namespace, prop=prop, - show=show, type=type, - toponly='1' if toponly else None)) - return listing.List(self, 'recentchanges', 'rc', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def revisions(self, revids, prop='ids|timestamp|flags|comment|user'): + kwargs = dict( + listing.List.generate_kwargs( + "rc", + start=start, + end=end, + dir=dir, + namespace=namespace, + prop=prop, + show=show, + type=type, + toponly="1" if toponly else None, + ) + ) + return listing.List(self, "recentchanges", "rc", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def revisions(self, revids, prop="ids|timestamp|flags|comment|user"): """Get data about a list of revisions. See also the `Page.revisions()` method. @@ -1404,24 +1540,21 @@ def revisions(self, revids, prop='ids|timestamp|flags|comment|user'): Returns: A list of revisions """ - kwargs = { - 'prop': 'revisions', - 'rvprop': prop, - 'revids': '|'.join(map(str, revids)) - } + kwargs = {"prop": "revisions", "rvprop": prop, "revids": "|".join(map(str, revids))} revisions = [] - pages = self.get('query', **kwargs).get('query', {}).get('pages', {}).values() + pages = self.get("query", **kwargs).get("query", {}).get("pages", {}).values() for page in pages: - for revision in page.get('revisions', ()): - revision['pageid'] = page.get('pageid') - revision['pagetitle'] = page.get('title') - revision['timestamp'] = parse_timestamp(revision['timestamp']) + for revision in page.get("revisions", ()): + revision["pageid"] = page.get("pageid") + revision["pagetitle"] = page.get("title") + revision["timestamp"] = parse_timestamp(revision["timestamp"]) revisions.append(revision) return revisions - def search(self, search, namespace='0', what=None, redirects=False, limit=None, - max_items=None, api_chunk_size=None): + def search( + self, search, namespace="0", what=None, redirects=False, limit=None, max_items=None, api_chunk_size=None + ): """Perform a full text search. API doc: https://www.mediawiki.org/wiki/API:Search @@ -1447,39 +1580,62 @@ def search(self, search, namespace='0', what=None, redirects=False, limit=None, mwclient.listings.List: Search results iterator """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('sr', search=search, - namespace=namespace, what=what)) + kwargs = dict(listing.List.generate_kwargs("sr", search=search, namespace=namespace, what=what)) if redirects: - kwargs['srredirects'] = '1' - return listing.List(self, 'search', 'sr', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) - - def usercontributions(self, user, start=None, end=None, dir='older', namespace=None, - prop=None, show=None, limit=None, uselang=None, max_items=None, - api_chunk_size=None): + kwargs["srredirects"] = "1" + return listing.List(self, "search", "sr", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) + + def usercontributions( + self, + user, + start=None, + end=None, + dir="older", + namespace=None, + prop=None, + show=None, + limit=None, + uselang=None, + max_items=None, + api_chunk_size=None, + ): """ List the contributions made by a given user to the wiki. API doc: https://www.mediawiki.org/wiki/API:Usercontribs """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('uc', user=user, start=start, end=end, - dir=dir, namespace=namespace, - prop=prop, show=show)) - return listing.List(self, 'usercontribs', 'uc', max_items=max_items, - api_chunk_size=api_chunk_size, uselang=uselang, **kwargs) + kwargs = dict( + listing.List.generate_kwargs( + "uc", user=user, start=start, end=end, dir=dir, namespace=namespace, prop=prop, show=show + ) + ) + return listing.List( + self, "usercontribs", "uc", max_items=max_items, api_chunk_size=api_chunk_size, uselang=uselang, **kwargs + ) - def users(self, users, prop='blockinfo|groups|editcount'): + def users(self, users, prop="blockinfo|groups|editcount"): """ Get information about a list of users. API doc: https://www.mediawiki.org/wiki/API:Users """ - return listing.List(self, 'users', 'us', ususers='|'.join(users), usprop=prop) - - def watchlist(self, allrev=False, start=None, end=None, namespace=None, dir='older', - prop=None, show=None, limit=None, max_items=None, api_chunk_size=None): + return listing.List(self, "users", "us", ususers="|".join(users), usprop=prop) + + def watchlist( + self, + allrev=False, + start=None, + end=None, + namespace=None, + dir="older", + prop=None, + show=None, + limit=None, + max_items=None, + api_chunk_size=None, + ): """ List the pages on the current user's watchlist. @@ -1487,13 +1643,12 @@ def watchlist(self, allrev=False, start=None, end=None, namespace=None, dir='old """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs('wl', start=start, end=end, - namespace=namespace, dir=dir, - prop=prop, show=show)) + kwargs = dict( + listing.List.generate_kwargs("wl", start=start, end=end, namespace=namespace, dir=dir, prop=prop, show=show) + ) if allrev: - kwargs['wlallrev'] = '1' - return listing.List(self, 'watchlist', 'wl', max_items=max_items, - api_chunk_size=api_chunk_size, **kwargs) + kwargs["wlallrev"] = "1" + return listing.List(self, "watchlist", "wl", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs) def expandtemplates(self, text, title=None, generatexml=False): """ @@ -1509,17 +1664,17 @@ def expandtemplates(self, text, title=None, generatexml=False): kwargs = {} if title is not None: - kwargs['title'] = title + kwargs["title"] = title if generatexml: # FIXME: Deprecated and replaced by `prop=parsetree`. - kwargs['generatexml'] = '1' + kwargs["generatexml"] = "1" - result = self.post('expandtemplates', text=text, **kwargs) + result = self.post("expandtemplates", text=text, **kwargs) if generatexml: - return result['expandtemplates']['*'], result['parsetree']['*'] + return result["expandtemplates"]["*"], result["parsetree"]["*"] else: - return result['expandtemplates']['*'] + return result["expandtemplates"]["*"] def ask(self, query, title=None): """ @@ -1545,15 +1700,16 @@ def ask(self, query, title=None): """ kwargs = {} if title is None: - kwargs['title'] = title + kwargs["title"] = title offset = 0 while offset is not None: - results = self.raw_api('ask', query='{query}|offset={offset}'.format( - query=query, offset=offset), http_method='GET', **kwargs) + results = self.raw_api( + "ask", query="{query}|offset={offset}".format(query=query, offset=offset), http_method="GET", **kwargs + ) self.handle_api_result(results) # raises APIError on error - offset = results.get('query-continue-offset') - answers = results['query'].get('results', []) + offset = results.get("query-continue-offset") + answers = results["query"].get("results", []) if isinstance(answers, dict): # In older versions of Semantic MediaWiki (at least until 2.3.0) diff --git a/newapi/super/mwclient/errors.py b/newapi/super/mwclient/errors.py index 5c87cca..963f015 100644 --- a/newapi/super/mwclient/errors.py +++ b/newapi/super/mwclient/errors.py @@ -58,17 +58,13 @@ def __init__(self, file_name): self.file_name = file_name def __str__(self): - return ('The file "{0}" already exists. Set ignore=True to overwrite it.' - .format(self.file_name)) + return 'The file "{0}" already exists. Set ignore=True to overwrite it.'.format(self.file_name) class LoginError(MwClientError): def __init__(self, site, code, info): - super(LoginError, self).__init__( - site, - {'result': code, 'reason': info} # For backwards-compability - ) + super(LoginError, self).__init__(site, {"result": code, "reason": info}) # For backwards-compability self.site = site self.code = code self.info = info @@ -84,11 +80,13 @@ class OAuthAuthorizationError(LoginError): class AssertUserFailedError(MwClientError): def __init__(self): - super(AssertUserFailedError, self).__init__(( - 'By default, mwclient protects you from accidentally editing ' - 'without being logged in. If you actually want to edit without ' - 'logging in, you can set force_login on the Site object to False.' - )) + super(AssertUserFailedError, self).__init__( + ( + "By default, mwclient protects you from accidentally editing " + "without being logged in. If you actually want to edit without " + "logging in, you can set force_login on the Site object to False." + ) + ) def __str__(self): return self.args[0] @@ -109,11 +107,13 @@ class NoWriteApi(MwClientError): class InvalidResponse(MwClientError): def __init__(self, response_text=None): - super(InvalidResponse, self).__init__(( - 'Did not get a valid JSON response from the server. Check that ' - 'you used the correct hostname. If you did, the server might ' - 'be wrongly configured or experiencing temporary problems.'), - response_text + super(InvalidResponse, self).__init__( + ( + "Did not get a valid JSON response from the server. Check that " + "you used the correct hostname. If you did, the server might " + "be wrongly configured or experiencing temporary problems." + ), + response_text, ) self.response_text = response_text diff --git a/newapi/super/mwclient/image.py b/newapi/super/mwclient/image.py index f3aef5b..6de17a8 100644 --- a/newapi/super/mwclient/image.py +++ b/newapi/super/mwclient/image.py @@ -1,21 +1,20 @@ +from . import listing, page from .util import handle_limit -from . import listing -from . import page class Image(page.Page): def __init__(self, site, name, info=None): super(Image, self).__init__( - site, name, info, extra_properties={ - 'imageinfo': ( - ('iiprop', - 'timestamp|user|comment|url|size|sha1|metadata|mime|archivename'), - ) - } + site, + name, + info, + extra_properties={ + "imageinfo": (("iiprop", "timestamp|user|comment|url|size|sha1|metadata|mime|archivename"),) + }, ) - self.imagerepository = self._info.get('imagerepository', '') - self.imageinfo = self._info.get('imageinfo', ({}, ))[0] + self.imagerepository = self._info.get("imagerepository", "") + self.imageinfo = self._info.get("imageinfo", ({},))[0] def imagehistory(self): """ @@ -24,32 +23,39 @@ def imagehistory(self): API doc: https://www.mediawiki.org/wiki/API:Imageinfo """ return listing.PageProperty( - self, 'imageinfo', 'ii', - iiprop='timestamp|user|comment|url|size|sha1|metadata|mime|archivename' + self, "imageinfo", "ii", iiprop="timestamp|user|comment|url|size|sha1|metadata|mime|archivename" ) - def imageusage(self, namespace=None, filterredir='all', redirect=False, - limit=None, generator=True, max_items=None, api_chunk_size=None): + def imageusage( + self, + namespace=None, + filterredir="all", + redirect=False, + limit=None, + generator=True, + max_items=None, + api_chunk_size=None, + ): """ List pages that use the given file. API doc: https://www.mediawiki.org/wiki/API:Imageusage """ - prefix = listing.List.get_prefix('iu', generator) - kwargs = dict(listing.List.generate_kwargs( - prefix, title=self.name, namespace=namespace, filterredir=filterredir - )) + prefix = listing.List.get_prefix("iu", generator) + kwargs = dict( + listing.List.generate_kwargs(prefix, title=self.name, namespace=namespace, filterredir=filterredir) + ) (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) if redirect: - kwargs['%sredirect' % prefix] = '1' + kwargs["%sredirect" % prefix] = "1" return listing.List.get_list(generator)( self.site, - 'imageusage', - 'iu', + "imageusage", + "iu", max_items=max_items, api_chunk_size=api_chunk_size, - return_values='title', - **kwargs + return_values="title", + **kwargs, ) def duplicatefiles(self, limit=None, max_items=None, api_chunk_size=None): @@ -62,13 +68,7 @@ def duplicatefiles(self, limit=None, max_items=None, api_chunk_size=None): not only specify the API chunk size. """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - return listing.PageProperty( - self, - 'duplicatefiles', - 'df', - max_items=max_items, - api_chunk_size=api_chunk_size - ) + return listing.PageProperty(self, "duplicatefiles", "df", max_items=max_items, api_chunk_size=api_chunk_size) def download(self, destination=None): """ @@ -84,7 +84,7 @@ def download(self, destination=None): Args: destination (file object): Destination file """ - url = self.imageinfo['url'] + url = self.imageinfo["url"] if destination is not None: res = self.site.connection.get(url, stream=True) for chunk in res.iter_content(1024): @@ -93,8 +93,4 @@ def download(self, destination=None): return self.site.connection.get(url).content def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.name, - self.site - ) + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) diff --git a/newapi/super/mwclient/listing.py b/newapi/super/mwclient/listing.py index 5626f28..d88a606 100644 --- a/newapi/super/mwclient/listing.py +++ b/newapi/super/mwclient/listing.py @@ -1,6 +1,5 @@ -from .util import parse_timestamp, handle_limit -from . import page -from . import image +from . import image, page +from .util import handle_limit, parse_timestamp class List: @@ -18,12 +17,21 @@ class List: to its misleading name. """ - def __init__(self, site, list_name, prefix, - limit=None, return_values=None, max_items=None, - api_chunk_size=None, *args, **kwargs): + def __init__( + self, + site, + list_name, + prefix, + limit=None, + return_values=None, + max_items=None, + api_chunk_size=None, + *args, + **kwargs, + ): self.site = site self.list_name = list_name - self.generator = 'list' + self.generator = "list" self.prefix = prefix kwargs.update(args) @@ -36,7 +44,7 @@ def __init__(self, site, list_name, prefix, # unneeded extra items (so long as it's below API limit) api_limit = site.api_limit api_chunk_size = api_chunk_size or min(max_items or api_limit, api_limit) - self.args[self.prefix + 'limit'] = str(api_chunk_size) + self.args[self.prefix + "limit"] = str(api_chunk_size) self.count = 0 self.max_items = max_items @@ -69,8 +77,8 @@ def __next__(self): self.load_chunk() self.count += 1 - if 'timestamp' in item: - item['timestamp'] = parse_timestamp(item['timestamp']) + if "timestamp" in item: + item["timestamp"] = parse_timestamp(item["timestamp"]) if isinstance(self, GeneratorList): return item @@ -96,45 +104,38 @@ def load_chunk(self): Else, set `self.last` to True. """ - data = self.site.get( - 'query', (self.generator, self.list_name), - *[(str(k), v) for k, v in self.args.items()] - ) + data = self.site.get("query", (self.generator, self.list_name), *[(str(k), v) for k, v in self.args.items()]) if not data: # Non existent page print(StopIteration) # Process response if not empty. # See: https://github.com/mwclient/mwclient/issues/194 - if 'query' in data: + if "query" in data: self.set_iter(data) - if data.get('continue'): + if data.get("continue"): # New style continuation, added in MediaWiki 1.21 - self.args.update(data['continue']) + self.args.update(data["continue"]) - elif self.list_name in data.get('query-continue', ()): + elif self.list_name in data.get("query-continue", ()): # Old style continuation - self.args.update(data['query-continue'][self.list_name]) + self.args.update(data["query-continue"][self.list_name]) else: self.last = True def set_iter(self, data): """Set `self._iter` to the API response `data`.""" - if self.result_member not in data['query']: + if self.result_member not in data["query"]: self._iter = iter(range(0)) - elif type(data['query'][self.result_member]) is list: - self._iter = iter(data['query'][self.result_member]) + elif type(data["query"][self.result_member]) is list: + self._iter = iter(data["query"][self.result_member]) else: - self._iter = iter(data['query'][self.result_member].values()) + self._iter = iter(data["query"][self.result_member].values()) def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.list_name, - self.site - ) + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.list_name, self.site) @staticmethod def generate_kwargs(_prefix, *args, **kwargs): @@ -145,7 +146,7 @@ def generate_kwargs(_prefix, *args, **kwargs): @staticmethod def get_prefix(prefix, generator=False): - return ('g' if generator else '') + prefix + return ("g" if generator else "") + prefix @staticmethod def get_list(generator=False): @@ -158,7 +159,7 @@ def __init__(self, nested_param, *args, **kwargs): self.nested_param = nested_param def set_iter(self, data): - self._iter = iter(data['query'][self.result_member][self.nested_param]) + self._iter = iter(data["query"][self.result_member][self.nested_param]) class GeneratorList(List): @@ -170,32 +171,31 @@ class GeneratorList(List): """ def __init__(self, site, list_name, prefix, *args, **kwargs): - super(GeneratorList, self).__init__(site, list_name, prefix, - *args, **kwargs) + super(GeneratorList, self).__init__(site, list_name, prefix, *args, **kwargs) - self.args['g' + self.prefix + 'limit'] = self.args[self.prefix + 'limit'] - del self.args[self.prefix + 'limit'] - self.generator = 'generator' + self.args["g" + self.prefix + "limit"] = self.args[self.prefix + "limit"] + del self.args[self.prefix + "limit"] + self.generator = "generator" - self.args['prop'] = 'info|imageinfo' - self.args['inprop'] = 'protection' + self.args["prop"] = "info|imageinfo" + self.args["inprop"] = "protection" - self.result_member = 'pages' + self.result_member = "pages" self.page_class = page.Page def __next__(self): info = super(GeneratorList, self).__next__() - if info['ns'] == 14: - return Category(self.site, '', info) - if info['ns'] == 6: - return image.Image(self.site, '', info) - return page.Page(self.site, '', info) + if info["ns"] == 14: + return Category(self.site, "", info) + if info["ns"] == 6: + return image.Image(self.site, "", info) + return page.Page(self.site, "", info) def load_chunk(self): # Put this here so that the constructor does not fail # on uninitialized sites - self.args['iiprop'] = 'timestamp|user|comment|url|size|sha1|metadata|archivename' + self.args["iiprop"] = "timestamp|user|comment|url|size|sha1|metadata|archivename" return super(GeneratorList, self).load_chunk() @@ -204,45 +204,42 @@ class Category(page.Page, GeneratorList): def __init__(self, site, name, info=None, namespace=None): page.Page.__init__(self, site, name, info) kwargs = {} - kwargs['gcmtitle'] = self.name + kwargs["gcmtitle"] = self.name if namespace: - kwargs['gcmnamespace'] = namespace - GeneratorList.__init__(self, site, 'categorymembers', 'cm', **kwargs) + kwargs["gcmnamespace"] = namespace + GeneratorList.__init__(self, site, "categorymembers", "cm", **kwargs) def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.name, - self.site + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) + + def members( + self, prop="ids|title", namespace=None, sort="sortkey", dir="asc", start=None, end=None, generator=True + ): + prefix = self.get_prefix("cm", generator) + kwargs = dict( + self.generate_kwargs( + prefix, prop=prop, namespace=namespace, sort=sort, dir=dir, start=start, end=end, title=self.name + ) ) - - def members(self, prop='ids|title', namespace=None, sort='sortkey', - dir='asc', start=None, end=None, generator=True): - prefix = self.get_prefix('cm', generator) - kwargs = dict(self.generate_kwargs(prefix, prop=prop, namespace=namespace, - sort=sort, dir=dir, start=start, end=end, - title=self.name)) - return self.get_list(generator)(self.site, 'categorymembers', 'cm', **kwargs) + return self.get_list(generator)(self.site, "categorymembers", "cm", **kwargs) class PageList(GeneratorList): - def __init__(self, site, prefix=None, start=None, namespace=0, redirects='all', - end=None): + def __init__(self, site, prefix=None, start=None, namespace=0, redirects="all", end=None): self.namespace = namespace kwargs = {} if prefix: - kwargs['gapprefix'] = prefix + kwargs["gapprefix"] = prefix if start: - kwargs['gapfrom'] = start + kwargs["gapfrom"] = start if end: - kwargs['gapto'] = end + kwargs["gapto"] = end - super(PageList, self).__init__(site, 'allpages', 'ap', - gapnamespace=str(namespace), - gapfilterredir=redirects, - **kwargs) + super(PageList, self).__init__( + site, "allpages", "ap", gapnamespace=str(namespace), gapfilterredir=redirects, **kwargs + ) def __getitem__(self, name): return self.get(name, None) @@ -258,7 +255,7 @@ def get(self, name, info=()): One of Category, Image or Page (default), according to namespace. """ if self.namespace != 0: - full_page_name = u"{namespace}:{name}".format( + full_page_name = "{namespace}:{name}".format( namespace=self.site.namespaces[self.namespace], name=name, ) @@ -293,11 +290,11 @@ def guess_namespace(self, name): for ns in self.site.namespaces: if ns == 0: continue - namespace = '%s:' % self.site.namespaces[ns].replace(' ', '_') + namespace = "%s:" % self.site.namespaces[ns].replace(" ", "_") if name.startswith(namespace): return ns elif ns in self.site.default_namespaces: - namespace = '%s:' % self.site.default_namespaces[ns].replace(' ', '_') + namespace = "%s:" % self.site.default_namespaces[ns].replace(" ", "_") if name.startswith(namespace): return ns return 0 @@ -306,15 +303,13 @@ def guess_namespace(self, name): class PageProperty(List): def __init__(self, page, prop, prefix, *args, **kwargs): - super(PageProperty, self).__init__(page.site, prop, prefix, - titles=page.name, - *args, **kwargs) + super(PageProperty, self).__init__(page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page - self.generator = 'prop' + self.generator = "prop" def set_iter(self, data): - for page in data['query']['pages'].values(): - if page['title'] == self.page.name: + for page in data["query"]["pages"].values(): + if page["title"] == self.page.name: self._iter = iter(page.get(self.list_name, ())) return print(StopIteration) @@ -323,15 +318,13 @@ def set_iter(self, data): class PagePropertyGenerator(GeneratorList): def __init__(self, page, prop, prefix, *args, **kwargs): - super(PagePropertyGenerator, self).__init__(page.site, prop, prefix, - titles=page.name, - *args, **kwargs) + super(PagePropertyGenerator, self).__init__(page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page class RevisionsIterator(PageProperty): def load_chunk(self): - if 'rvstartid' in self.args and 'rvstart' in self.args: - del self.args['rvstart'] + if "rvstartid" in self.args and "rvstart" in self.args: + del self.args["rvstart"] return super(RevisionsIterator, self).load_chunk() diff --git a/newapi/super/mwclient/page.py b/newapi/super/mwclient/page.py index 04080e7..fa66b0d 100644 --- a/newapi/super/mwclient/page.py +++ b/newapi/super/mwclient/page.py @@ -1,8 +1,8 @@ import time -from .util import parse_timestamp, handle_limit -from . import listing -from . import errors +from . import errors, listing +from .util import handle_limit, parse_timestamp + class Page: @@ -16,67 +16,61 @@ def __init__(self, site, name, info=None, extra_properties=None): if not info: if extra_properties: - prop = 'info|' + '|'.join(extra_properties.keys()) + prop = "info|" + "|".join(extra_properties.keys()) extra_props = [] for extra_prop in extra_properties.values(): extra_props.extend(extra_prop) else: - prop = 'info' + prop = "info" extra_props = () if type(name) is int: - info = self.site.get('query', prop=prop, pageids=name, - inprop='protection', *extra_props) + info = self.site.get("query", prop=prop, pageids=name, inprop="protection", *extra_props) else: - info = self.site.get('query', prop=prop, titles=name, - inprop='protection', *extra_props) - info = next(iter(info['query']['pages'].values())) + info = self.site.get("query", prop=prop, titles=name, inprop="protection", *extra_props) + info = next(iter(info["query"]["pages"].values())) self._info = info - if 'invalid' in info: - print(errors.InvalidPageTitle(info.get('invalidreason'))) + if "invalid" in info: + print(errors.InvalidPageTitle(info.get("invalidreason"))) - self.namespace = info.get('ns', 0) - self.name = info.get('title', '') + self.namespace = info.get("ns", 0) + self.name = info.get("title", "") if self.namespace: self.page_title = self.strip_namespace(self.name) else: self.page_title = self.name - self.base_title = self.page_title.split('/')[0] - self.base_name = self.name.split('/')[0] - - self.touched = parse_timestamp(info.get('touched')) - self.revision = info.get('lastrevid', 0) - self.exists = 'missing' not in info - self.length = info.get('length') - self.protection = { - i['type']: (i['level'], i.get('expiry')) - for i in info.get('protection', ()) - if i - } - self.redirect = 'redirect' in info - self.pageid = info.get('pageid', None) - self.contentmodel = info.get('contentmodel', None) - self.pagelanguage = info.get('pagelanguage', None) - self.restrictiontypes = info.get('restrictiontypes', None) + self.base_title = self.page_title.split("/")[0] + self.base_name = self.name.split("/")[0] + + self.touched = parse_timestamp(info.get("touched")) + self.revision = info.get("lastrevid", 0) + self.exists = "missing" not in info + self.length = info.get("length") + self.protection = {i["type"]: (i["level"], i.get("expiry")) for i in info.get("protection", ()) if i} + self.redirect = "redirect" in info + self.pageid = info.get("pageid", None) + self.contentmodel = info.get("contentmodel", None) + self.pagelanguage = info.get("pagelanguage", None) + self.restrictiontypes = info.get("restrictiontypes", None) self.last_rev_time = None self.edit_time = None def redirects_to(self): - """ Get the redirect target page, or None if the page is not a redirect.""" - info = self.site.get('query', prop='pageprops', titles=self.name, redirects='') - if 'redirects' in info['query']: - for page in info['query']['redirects']: - if page['from'] == self.name: - return Page(self.site, page['to']) + """Get the redirect target page, or None if the page is not a redirect.""" + info = self.site.get("query", prop="pageprops", titles=self.name, redirects="") + if "redirects" in info["query"]: + for page in info["query"]["redirects"]: + if page["from"] == self.name: + return Page(self.site, page["to"]) return None else: return None def resolve_redirect(self): - """ Get the redirect target page, or the current page if its not a redirect.""" + """Get the redirect target page, or the current page if its not a redirect.""" target_page = self.redirects_to() if target_page is None: return self @@ -84,26 +78,22 @@ def resolve_redirect(self): return target_page def __repr__(self): - return "<%s object '%s' for %s>" % ( - self.__class__.__name__, - self.name, - self.site - ) + return "<%s object '%s' for %s>" % (self.__class__.__name__, self.name, self.site) @staticmethod def strip_namespace(title): - if title[0] == ':': + if title[0] == ":": title = title[1:] - return title[title.find(':') + 1:] + return title[title.find(":") + 1 :] @staticmethod def normalize_title(title): # TODO: Make site dependent title = title.strip() - if title[0] == ':': + if title[0] == ":": title = title[1:] title = title[0].upper() + title[1:] - title = title.replace(' ', '_') + title = title.replace(" ", "_") return title def can(self, action): @@ -116,15 +106,15 @@ def can(self, action): """ level = self.protection.get(action, (action,))[0] - if level == 'sysop': - level = 'editprotected' + if level == "sysop": + level = "editprotected" return level in self.site.rights def get_token(self, type, force=False): return self.site.get_token(type, force, title=self.name) - def text(self, section=None, expandtemplates=False, cache=True, slot='main'): + def text(self, section=None, expandtemplates=False, cache=True, slot="main"): """Get the current wikitext of the page, or of a specific section. If the page does not exist, an empty string is returned. By @@ -139,10 +129,10 @@ def text(self, section=None, expandtemplates=False, cache=True, slot='main'): cache (bool): Use in-memory caching (default: `True`) """ - if not self.can('read'): + if not self.can("read"): print(errors.InsufficientPermission(self)) if not self.exists: - return '' + return "" if section is not None: section = str(section) @@ -153,17 +143,16 @@ def text(self, section=None, expandtemplates=False, cache=True, slot='main'): # we set api_chunk_size not max_items because otherwise revisions' # default api_chunk_size of 50 gets used and we get 50 revisions; # no need to set max_items as well as we only iterate one time - revs = self.revisions(prop='content|timestamp', api_chunk_size=1, section=section, - slots=slot) + revs = self.revisions(prop="content|timestamp", api_chunk_size=1, section=section, slots=slot) try: rev = next(revs) - if 'slots' in rev: - text = rev['slots'][slot]['*'] + if "slots" in rev: + text = rev["slots"][slot]["*"] else: - text = rev['*'] - self.last_rev_time = rev['timestamp'] + text = rev["*"] + self.last_rev_time = rev["timestamp"] except StopIteration: - text = '' + text = "" self.last_rev_time = None if not expandtemplates: self.edit_time = time.gmtime() @@ -180,21 +169,16 @@ def save(self, *args, **kwargs): """Alias for edit, for maintaining backwards compatibility.""" return self.edit(*args, **kwargs) - def edit(self, text, summary='', minor=False, bot=True, section=None, **kwargs): - """Update the text of a section or the whole page by performing an edit operation. - """ + def edit(self, text, summary="", minor=False, bot=True, section=None, **kwargs): + """Update the text of a section or the whole page by performing an edit operation.""" return self._edit(summary, minor, bot, section, text=text, **kwargs) - def append(self, text, summary='', minor=False, bot=True, section=None, - **kwargs): - """Append text to a section or the whole page by performing an edit operation. - """ + def append(self, text, summary="", minor=False, bot=True, section=None, **kwargs): + """Append text to a section or the whole page by performing an edit operation.""" return self._edit(summary, minor, bot, section, appendtext=text, **kwargs) - def prepend(self, text, summary='', minor=False, bot=True, section=None, - **kwargs): - """Prepend text to a section or the whole page by performing an edit operation. - """ + def prepend(self, text, summary="", minor=False, bot=True, section=None, **kwargs): + """Prepend text to a section or the whole page by performing an edit operation.""" return self._edit(summary, minor, bot, section, prependtext=text, **kwargs) def _edit(self, summary, minor, bot, section, **kwargs): @@ -202,42 +186,40 @@ def _edit(self, summary, minor, bot, section, **kwargs): print(errors.AssertUserFailedError()) if self.site.blocked: print(errors.UserBlocked(self.site.blocked)) - if not self.can('edit'): + if not self.can("edit"): print(errors.ProtectedPageError(self)) data = {} if minor: - data['minor'] = '1' + data["minor"] = "1" if not minor: - data['notminor'] = '1' + data["notminor"] = "1" if self.last_rev_time: - data['basetimestamp'] = time.strftime('%Y%m%d%H%M%S', self.last_rev_time) + data["basetimestamp"] = time.strftime("%Y%m%d%H%M%S", self.last_rev_time) if self.edit_time: - data['starttimestamp'] = time.strftime('%Y%m%d%H%M%S', self.edit_time) + data["starttimestamp"] = time.strftime("%Y%m%d%H%M%S", self.edit_time) if bot: - data['bot'] = '1' + data["bot"] = "1" if section is not None: - data['section'] = section + data["section"] = section data.update(kwargs) if self.site.force_login: - data['assert'] = 'user' + data["assert"] = "user" def do_edit(): - result = self.site.post('edit', title=self.name, summary=summary, - token=self.get_token('edit'), - **data) - if result['edit'].get('result').lower() == 'failure': - print(errors.EditError(self, result['edit'])) + result = self.site.post("edit", title=self.name, summary=summary, token=self.get_token("edit"), **data) + if result["edit"].get("result").lower() == "failure": + print(errors.EditError(self, result["edit"])) return result try: result = do_edit() except errors.APIError as e: - if e.code == 'badtoken': + if e.code == "badtoken": # Retry, but only once to avoid an infinite loop - self.get_token('edit', force=True) + self.get_token("edit", force=True) try: result = do_edit() except errors.APIError as e: @@ -246,29 +228,37 @@ def do_edit(): self.handle_edit_error(e, summary) # 'newtimestamp' is not included if no change was made - if 'newtimestamp' in result['edit'].keys(): - self.last_rev_time = parse_timestamp(result['edit'].get('newtimestamp')) + if "newtimestamp" in result["edit"].keys(): + self.last_rev_time = parse_timestamp(result["edit"].get("newtimestamp")) # Workaround for https://phabricator.wikimedia.org/T211233 for cookie in self.site.connection.cookies: - if 'PostEditRevision' in cookie.name: - self.site.connection.cookies.clear(cookie.domain, cookie.path, - cookie.name) + if "PostEditRevision" in cookie.name: + self.site.connection.cookies.clear(cookie.domain, cookie.path, cookie.name) # clear the page text cache self._textcache = {} - return result['edit'] + return result["edit"] def handle_edit_error(self, e, summary): - if e.code == 'editconflict': + if e.code == "editconflict": print(errors.EditError(self, summary, e.info)) - elif e.code in {'protectedtitle', 'cantcreate', 'cantcreate-anon', - 'noimageredirect-anon', 'noimageredirect', 'noedit-anon', - 'noedit', 'protectedpage', 'cascadeprotected', - 'customcssjsprotected', - 'protectednamespace-interface', 'protectednamespace'}: + elif e.code in { + "protectedtitle", + "cantcreate", + "cantcreate-anon", + "noimageredirect-anon", + "noimageredirect", + "noedit-anon", + "noedit", + "protectedpage", + "cascadeprotected", + "customcssjsprotected", + "protectednamespace-interface", + "protectednamespace", + }: print(errors.ProtectedPageError(self, e.code, e.info)) - elif e.code == 'assertuserfailed': + elif e.code == "assertuserfailed": print(errors.AssertUserFailedError()) else: print(e) @@ -281,10 +271,9 @@ def touch(self): """ if not self.exists: return - self.append('') + self.append("") - def move(self, new_title, reason='', move_talk=True, no_redirect=False, - move_subpages=False, ignore_warnings=False): + def move(self, new_title, reason="", move_talk=True, no_redirect=False, move_subpages=False, ignore_warnings=False): """Move (rename) page to new_title. If user account is an administrator, specify no_redirect as True to not @@ -294,73 +283,89 @@ def move(self, new_title, reason='', move_talk=True, no_redirect=False, exception is raised. """ - if not self.can('move'): + if not self.can("move"): print(errors.InsufficientPermission(self)) data = {} if move_talk: - data['movetalk'] = '1' + data["movetalk"] = "1" if no_redirect: - data['noredirect'] = '1' + data["noredirect"] = "1" if move_subpages: - data['movesubpages'] = '1' + data["movesubpages"] = "1" if ignore_warnings: - data['ignorewarnings'] = '1' - result = self.site.post('move', ('from', self.name), to=new_title, - token=self.get_token('move'), reason=reason, **data) - return result['move'] + data["ignorewarnings"] = "1" + result = self.site.post( + "move", ("from", self.name), to=new_title, token=self.get_token("move"), reason=reason, **data + ) + return result["move"] - def delete(self, reason='', watch=False, unwatch=False, oldimage=False): + def delete(self, reason="", watch=False, unwatch=False, oldimage=False): """Delete page. If user does not have permission to delete page, an InsufficientPermission exception is raised. """ - if not self.can('delete'): + if not self.can("delete"): print(errors.InsufficientPermission(self)) data = {} if watch: - data['watch'] = '1' + data["watch"] = "1" if unwatch: - data['unwatch'] = '1' + data["unwatch"] = "1" if oldimage: - data['oldimage'] = oldimage - result = self.site.post('delete', title=self.name, - token=self.get_token('delete'), - reason=reason, **data) - return result['delete'] + data["oldimage"] = oldimage + result = self.site.post("delete", title=self.name, token=self.get_token("delete"), reason=reason, **data) + return result["delete"] def purge(self): """Purge server-side cache of page. This will re-render templates and other dynamic content. """ - self.site.post('purge', titles=self.name) + self.site.post("purge", titles=self.name) # def watch: requires 1.14 # Properties - def backlinks(self, namespace=None, filterredir='all', redirect=False, - limit=None, generator=True, max_items=None, api_chunk_size=None): + def backlinks( + self, + namespace=None, + filterredir="all", + redirect=False, + limit=None, + generator=True, + max_items=None, + api_chunk_size=None, + ): """List pages that link to the current page, similar to Special:Whatlinkshere. API doc: https://www.mediawiki.org/wiki/API:Backlinks """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - prefix = listing.List.get_prefix('bl', generator) - kwargs = dict(listing.List.generate_kwargs( - prefix, namespace=namespace, filterredir=filterredir, - )) + prefix = listing.List.get_prefix("bl", generator) + kwargs = dict( + listing.List.generate_kwargs( + prefix, + namespace=namespace, + filterredir=filterredir, + ) + ) if redirect: - kwargs['%sredirect' % prefix] = '1' - kwargs[prefix + 'title'] = self.name + kwargs["%sredirect" % prefix] = "1" + kwargs[prefix + "title"] = self.name return listing.List.get_list(generator)( - self.site, 'backlinks', 'bl', max_items=max_items, - api_chunk_size=api_chunk_size, return_values='title', **kwargs + self.site, + "backlinks", + "bl", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="title", + **kwargs, ) def categories(self, generator=True, show=None): @@ -376,23 +381,18 @@ def categories(self, generator=True, show=None): Returns: listings.PagePropertyGenerator """ - prefix = listing.List.get_prefix('cl', generator) - kwargs = dict(listing.List.generate_kwargs( - prefix, show=show - )) + prefix = listing.List.get_prefix("cl", generator) + kwargs = dict(listing.List.generate_kwargs(prefix, show=show)) if generator: - return listing.PagePropertyGenerator( - self, 'categories', 'cl', **kwargs - ) + return listing.PagePropertyGenerator(self, "categories", "cl", **kwargs) else: # TODO: return sortkey if wanted - return listing.PageProperty( - self, 'categories', 'cl', return_values='title', **kwargs - ) + return listing.PageProperty(self, "categories", "cl", return_values="title", **kwargs) - def embeddedin(self, namespace=None, filterredir='all', limit=None, generator=True, - max_items=None, api_chunk_size=None): + def embeddedin( + self, namespace=None, filterredir="all", limit=None, generator=True, max_items=None, api_chunk_size=None + ): """List pages that transclude the current page. API doc: https://www.mediawiki.org/wiki/API:Embeddedin @@ -410,14 +410,18 @@ def embeddedin(self, namespace=None, filterredir='all', limit=None, generator=Tr listings.List: Page iterator """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - prefix = listing.List.get_prefix('ei', generator) - kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace, - filterredir=filterredir)) - kwargs[prefix + 'title'] = self.name + prefix = listing.List.get_prefix("ei", generator) + kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace, filterredir=filterredir)) + kwargs[prefix + "title"] = self.name return listing.List.get_list(generator)( - self.site, 'embeddedin', 'ei', max_items=max_items, - api_chunk_size=api_chunk_size, return_values='title', **kwargs + self.site, + "embeddedin", + "ei", + max_items=max_items, + api_chunk_size=api_chunk_size, + return_values="title", + **kwargs, ) def extlinks(self): @@ -426,7 +430,7 @@ def extlinks(self): API doc: https://www.mediawiki.org/wiki/API:Extlinks """ - return listing.PageProperty(self, 'extlinks', 'el', return_values='*') + return listing.PageProperty(self, "extlinks", "el", return_values="*") def images(self, generator=True): """List files/images embedded in the current page. @@ -435,10 +439,9 @@ def images(self, generator=True): """ if generator: - return listing.PagePropertyGenerator(self, 'images', '') + return listing.PagePropertyGenerator(self, "images", "") else: - return listing.PageProperty(self, 'images', '', - return_values='title') + return listing.PageProperty(self, "images", "", return_values="title") def iwlinks(self): """List interwiki links from the current page. @@ -446,8 +449,7 @@ def iwlinks(self): API doc: https://www.mediawiki.org/wiki/API:Iwlinks """ - return listing.PageProperty(self, 'iwlinks', 'iw', - return_values=('prefix', '*')) + return listing.PageProperty(self, "iwlinks", "iw", return_values=("prefix", "*")) def langlinks(self, **kwargs): """List interlanguage links from the current page. @@ -455,9 +457,7 @@ def langlinks(self, **kwargs): API doc: https://www.mediawiki.org/wiki/API:Langlinks """ - return listing.PageProperty(self, 'langlinks', 'll', - return_values=('lang', '*'), - **kwargs) + return listing.PageProperty(self, "langlinks", "ll", return_values=("lang", "*"), **kwargs) def links(self, namespace=None, generator=True, redirects=False): """List links to other pages from the current page. @@ -465,23 +465,35 @@ def links(self, namespace=None, generator=True, redirects=False): API doc: https://www.mediawiki.org/wiki/API:Links """ - prefix = listing.List.get_prefix('pl', generator) + prefix = listing.List.get_prefix("pl", generator) kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace)) if redirects: - kwargs['redirects'] = '1' + kwargs["redirects"] = "1" if generator: - return listing.PagePropertyGenerator(self, 'links', 'pl', **kwargs) + return listing.PagePropertyGenerator(self, "links", "pl", **kwargs) else: - return listing.PageProperty(self, 'links', 'pl', - return_values='title', **kwargs) - - def revisions(self, startid=None, endid=None, start=None, end=None, - dir='older', user=None, excludeuser=None, limit=None, - prop='ids|timestamp|flags|comment|user', - expandtemplates=False, section=None, - diffto=None, slots=None, uselang=None, max_items=None, - api_chunk_size=50): + return listing.PageProperty(self, "links", "pl", return_values="title", **kwargs) + + def revisions( + self, + startid=None, + endid=None, + start=None, + end=None, + dir="older", + user=None, + excludeuser=None, + limit=None, + prop="ids|timestamp|flags|comment|user", + expandtemplates=False, + section=None, + diffto=None, + slots=None, + uselang=None, + max_items=None, + api_chunk_size=50, + ): """List revisions of the current page. API doc: https://www.mediawiki.org/wiki/API:Revisions @@ -512,27 +524,35 @@ def revisions(self, startid=None, endid=None, start=None, end=None, listings.List: Revision iterator """ (max_items, api_chunk_size) = handle_limit(limit, max_items, api_chunk_size) - kwargs = dict(listing.List.generate_kwargs( - 'rv', startid=startid, endid=endid, start=start, end=end, user=user, - excludeuser=excludeuser, diffto=diffto, slots=slots - )) + kwargs = dict( + listing.List.generate_kwargs( + "rv", + startid=startid, + endid=endid, + start=start, + end=end, + user=user, + excludeuser=excludeuser, + diffto=diffto, + slots=slots, + ) + ) - if self.site.version[:2] < (1, 32) and 'rvslots' in kwargs: + if self.site.version[:2] < (1, 32) and "rvslots" in kwargs: # https://github.com/mwclient/mwclient/issues/199 - del kwargs['rvslots'] + del kwargs["rvslots"] - kwargs['rvdir'] = dir - kwargs['rvprop'] = prop - kwargs['uselang'] = uselang + kwargs["rvdir"] = dir + kwargs["rvprop"] = prop + kwargs["uselang"] = uselang if expandtemplates: - kwargs['rvexpandtemplates'] = '1' + kwargs["rvexpandtemplates"] = "1" if section is not None: - kwargs['rvsection'] = section + kwargs["rvsection"] = section - return listing.RevisionsIterator(self, 'revisions', 'rv', - max_items=max_items, - api_chunk_size=api_chunk_size, - **kwargs) + return listing.RevisionsIterator( + self, "revisions", "rv", max_items=max_items, api_chunk_size=api_chunk_size, **kwargs + ) def templates(self, namespace=None, generator=True): """List templates used on the current page. @@ -540,11 +560,9 @@ def templates(self, namespace=None, generator=True): API doc: https://www.mediawiki.org/wiki/API:Templates """ - prefix = listing.List.get_prefix('tl', generator) + prefix = listing.List.get_prefix("tl", generator) kwargs = dict(listing.List.generate_kwargs(prefix, namespace=namespace)) if generator: - return listing.PagePropertyGenerator(self, 'templates', prefix, - **kwargs) + return listing.PagePropertyGenerator(self, "templates", prefix, **kwargs) else: - return listing.PageProperty(self, 'templates', prefix, - return_values='title', **kwargs) + return listing.PageProperty(self, "templates", prefix, return_values="title", **kwargs) diff --git a/newapi/super/mwclient/sleep.py b/newapi/super/mwclient/sleep.py index d378f9e..73c9ab8 100644 --- a/newapi/super/mwclient/sleep.py +++ b/newapi/super/mwclient/sleep.py @@ -1,5 +1,6 @@ -import time import logging +import time + from .errors import MaximumRetriesExceeded log = logging.getLogger(__name__) @@ -25,6 +26,7 @@ class Sleepers: retry_timeout (int): The time to sleep for each past retry. callback (callable): A callable to be called on each retry. """ + def __init__(self, max_retries, retry_timeout, callback=lambda *x: None): self.max_retries = max_retries self.retry_timeout = retry_timeout @@ -59,6 +61,7 @@ class Sleeper: retry_timeout (int): The time to sleep for each past retry. callback (callable): A callable to be called on each retry. """ + def __init__(self, args, max_retries, retry_timeout, callback): self.args = args self.retries = 0 @@ -85,6 +88,6 @@ def sleep(self, min_time=0): if timeout < min_time: timeout = min_time - print(f'mwclient/sleep.py: Sleeping for {timeout} seconds') + print(f"mwclient/sleep.py: Sleeping for {timeout} seconds") # time.sleep(timeout) diff --git a/newapi/super/mwclient/util.py b/newapi/super/mwclient/util.py index 9f02705..fe6d0ec 100644 --- a/newapi/super/mwclient/util.py +++ b/newapi/super/mwclient/util.py @@ -1,5 +1,5 @@ -import time import io +import time import warnings @@ -12,9 +12,9 @@ def parse_timestamp(t): Returns: time.struct_time: A timestamp. """ - if t is None or t == '0000-00-00T00:00:00Z': + if t is None or t == "0000-00-00T00:00:00Z": return time.struct_time((0, 0, 0, 0, 0, 0, 0, 0, 0)) - return time.strptime(t, '%Y-%m-%dT%H:%M:%SZ') + return time.strptime(t, "%Y-%m-%dT%H:%M:%SZ") def read_in_chunks(stream, chunk_size): @@ -38,7 +38,7 @@ def handle_limit(limit, max_items, api_chunk_size): warnings.warn( "limit and api_chunk_size both specified, this is not supported! limit " "is deprecated, will use value of api_chunk_size", - DeprecationWarning + DeprecationWarning, ) else: warnings.warn( @@ -46,7 +46,7 @@ def handle_limit(limit, max_items, api_chunk_size): "api_chunk_size to set the number of items retrieved from the API at " "once, and/or max_items to limit the total number of items that will be " "yielded", - DeprecationWarning + DeprecationWarning, ) api_chunk_size = limit return (max_items, api_chunk_size) diff --git a/newapi/super/params_help.py b/newapi/super/params_help.py index 0abfb6d..d666b7a 100644 --- a/newapi/super/params_help.py +++ b/newapi/super/params_help.py @@ -3,8 +3,10 @@ from .super.params_help import PARAMS_HELPS """ -import sys + import json +import sys + from ..api_utils.except_err import exception_err @@ -18,7 +20,13 @@ def __init__(self) -> None: # pass def params_w(self, params) -> dict: - if self.family == "wikipedia" and self.lang == "ar" and params.get("summary") and self.username.find("bot") == -1 and "ibrahemsummary" not in sys.argv: + if ( + self.family == "wikipedia" + and self.lang == "ar" + and params.get("summary") + and self.username.find("bot") == -1 + and "ibrahemsummary" not in sys.argv + ): params["summary"] = "" self.Bot_or_himo = 1 if "bot" in self.username else 0 @@ -30,7 +38,11 @@ def params_w(self, params) -> dict: params["minor"] = self.Bot_or_himo if self.family != "toolforge": - if params["action"] in ["edit", "create", "upload", "delete", "move"] or params["action"].startswith("wb") or self.family == "wikidata": + if ( + params["action"] in ["edit", "create", "upload", "delete", "move"] + or params["action"].startswith("wb") + or self.family == "wikidata" + ): if "nologin" not in sys.argv and self.username: params["assertuser"] = self.username diff --git a/newapi/super/super_login.py b/newapi/super/super_login.py index 6fa3126..893dce6 100644 --- a/newapi/super/super_login.py +++ b/newapi/super/super_login.py @@ -19,9 +19,9 @@ import urllib.parse from ..api_utils import printe -from .handel_errors import HANDEL_ERRORS from ..api_utils.except_err import warn_err from ..api_utils.user_agent import default_user_agent +from .handel_errors import HANDEL_ERRORS # if "nomwclient" in sys.argv: # from .bot import LOGIN_HELPS @@ -37,6 +37,7 @@ ar_lag = {1: 3} urls_prints = {"all": 0} + class Login(LOGIN_HELPS, HANDEL_ERRORS): """ Represents a login session for a wiki. @@ -79,7 +80,11 @@ def p_url(self, params): no_url = ["lgpassword", "format"] no_remove = ["titles", "title"] # --- - pams2 = {k: v[:100] if isinstance(v, str) and len(v) > 100 and k not in no_remove else v for k, v in params.items() if k not in no_url} + pams2 = { + k: v[:100] if isinstance(v, str) and len(v) > 100 and k not in no_remove else v + for k, v in params.items() + if k not in no_url + } # --- self.url_o_print = f"{self.endpoint}?{urllib.parse.urlencode(pams2)}".replace("&format=json", "") # --- @@ -126,7 +131,11 @@ def filter_params(self, params): if self.family == "nccommons" and params.get("bot"): del params["bot"] - if "workibrahem" in sys.argv and "ibrahemsummary" not in sys.argv and params.get("summary", "").find("بوت:") != -1: + if ( + "workibrahem" in sys.argv + and "ibrahemsummary" not in sys.argv + and params.get("summary", "").find("بوت:") != -1 + ): params["summary"] = "" if params["action"] in ["query"]: @@ -226,7 +235,9 @@ def post_params(self, params, Type="get", addtoken=False, GET_CSRF=True, files=N return data - def post_continue(self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None): + def post_continue( + self, params, action, _p_="pages", p_empty=None, Max=500000, first=False, _p_2="", _p_2_empty=None + ): # --- printe.test_print("_______________________") printe.test_print(f"post_continue, start. {action=}, {_p_=}") diff --git a/newapi/toolforge_page.py b/newapi/toolforge_page.py deleted file mode 100644 index 61cba10..0000000 --- a/newapi/toolforge_page.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -""" -from .pages_bots.toolforge_page import ( - # bot_api, - # super_page, - catdepth_new, - MainPage, - add_User_table, - CatDepth, -) - -__all__ = [ - # "bot_api", - # "super_page", - "catdepth_new", - "MainPage", - "add_User_table", - "CatDepth", -] diff --git a/newapi/tt.py b/newapi/tt.py deleted file mode 100644 index 95677e0..0000000 --- a/newapi/tt.py +++ /dev/null @@ -1,25 +0,0 @@ -# -import os -from configparser import ConfigParser - -if os.getenv("HOME"): - project = "/data/project/himo" -else: - project = 'I:/core/bots/core1' - -db_connect_file = f"{project}/confs/db.ini" - -print(f"{db_connect_file=}") - -main_dir = os.path.dirname(__file__) -print(f"{main_dir=}") -print(f"{__file__=}") - -config2 = ConfigParser() -config2.read(db_connect_file) - -try: - user = config2["client"]["user"] - print(f"{user=}") -except KeyError as e: - print(f"error: {e}") diff --git a/newapi/wiki_page.py b/newapi/wiki_page.py deleted file mode 100644 index 7acd534..0000000 --- a/newapi/wiki_page.py +++ /dev/null @@ -1,21 +0,0 @@ -""" - - -""" -from .pages_bots.wiki_page import ( - home_dir, - MainPage, - user_agent, - NEW_API, - CatDepth, - change_codes -) - -__all__ = [ - "home_dir", - "user_agent", - "MainPage", - "NEW_API", - "CatDepth", - "change_codes", -] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cb836ea --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,147 @@ + +# ============================================ +# BLACK +# ============================================ +[tool.black] +line-length = 120 +target-version = ["py313"] +include = '\.pyi?$' +exclude = ''' + /( + \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ +''' + +# ============================================ +# ISORT +# ============================================ +[tool.isort] +profile = "black" +line_length = 120 + +# Keep vertical formatting (multi-line import style) +multi_line_output = 3 +use_parentheses = true +force_grid_wrap = 0 +include_trailing_comma = true + +# OPTIONAL: You can add this to ensure isort does NOT collapse into one line +ensure_newline_before_comments = true + +# Project paths +src_paths = "ArWikiCats" +known_first_party = "ArWikiCats" +skip = [ + ".env", + "env", + ".venv", + "venv", + "build", + "dist", + "__pycache__", + ".mypy_cache", + ".pytest_cache", + ".git", +] + +# ============================================ +# RUFF +# ============================================ +[tool.ruff] +exclude = [ + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "old", + "build", + "dist", + "site-packages", + "venv", +] +line-length = 120 +target-version = "py313" +fix = true + +[tool.ruff.format] +# Allow Ruff/Black to maintain vertical list formatting +skip-magic-trailing-comma = false + +# Keep Black-like behavior +quote-style = "double" +indent-style = "space" +line-ending = "auto" + +# Format code blocks inside docstrings (excellent for README-like docstrings) +docstring-code-format = true +docstring-code-line-length = 120 + +[tool.ruff.lint] +# "E402", "E225", "E226", "E227", "E228", "E252", "F841", "E224", "E203", "F401", +ignore = ["E501", "UP035", "UP006", "N806", "N999", "UP015", "I001", "N802", "UP007", "UP045"] +select = [ + "E", # pycodestyle (error) + "F", # pyflakes + "W", # pycodestyle (warning) + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "PIE", # flake8-pie + "PLE", # pylint error + "RUF100", # Unused noqa comments + "PGH004", # blanket noqa comments + "UP", # pyupgrade + "C4", # flake8-comprehensions + "SIM101", # merge duplicate isinstance calls + "SIM201", "SIM202", "SIM222", "SIM223", # flake8-simplify + "FURB168", # Prefer is operator over isinstance for None checks + "FURB169", # Do not use is comparison with type(None). Use None + "FURB187", # avoid list reverse copy + "FURB188", # use str.remove(pre|suf)fix + "ISC001", # implicitly concatenated string + "RET501", "RET502", # better return None handling +] +# ============================================ +# Flynt +# ============================================ +[tool.flynt] +# Keep it aligned with Black/Ruff +line_length = 120 + +[tool.mypy] +python_version = "3.13" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true +line-length = 120 + +[tool.pylint.messages_control] +max-line-length = 120 +disable = [ + "C0111", # missing-docstring + "C0103", # invalid-name + "R0913", # too-many-arguments +] diff --git a/tests/TestALL_APIS.py b/tests/TestALL_APIS.py index 7c6525e..ac6ecd6 100644 --- a/tests/TestALL_APIS.py +++ b/tests/TestALL_APIS.py @@ -1,32 +1,35 @@ -import pytest from unittest.mock import MagicMock, patch + +import pytest from newapi import ALL_APIS @pytest.fixture def mock_dependencies(): - with patch('newapi.pages_bots.all_apis.Login') as mock_login, \ - patch('newapi.pages_bots.all_apis.super_page.MainPage') as mock_main_page, \ - patch('newapi.pages_bots.all_apis.catdepth_new.subcatquery') as mock_subcatquery, \ - patch('newapi.pages_bots.all_apis.bot_api.NEW_API') as mock_new_api, \ - patch('newapi.pages_bots.all_apis.printe') as mock_printe: + with ( + patch("newapi.pages_bots.all_apis.Login") as mock_login, + patch("newapi.pages_bots.all_apis.super_page.MainPage") as mock_main_page, + patch("newapi.pages_bots.all_apis.catdepth_new.subcatquery") as mock_subcatquery, + patch("newapi.pages_bots.all_apis.bot_api.NEW_API") as mock_new_api, + patch("newapi.pages_bots.all_apis.printe") as mock_printe, + ): # Setup mock login instance mock_login_instance = MagicMock() mock_login.return_value = mock_login_instance yield { - 'Login': mock_login, - 'LoginInstance': mock_login_instance, - 'MainPage': mock_main_page, - 'subcatquery': mock_subcatquery, - 'NEW_API': mock_new_api, - 'printe': mock_printe + "Login": mock_login, + "LoginInstance": mock_login_instance, + "MainPage": mock_main_page, + "subcatquery": mock_subcatquery, + "NEW_API": mock_new_api, + "printe": mock_printe, } def test_all_apis_init(mock_dependencies): - lang, family, username, password = 'en', 'wikipedia', 'user', 'pass' + lang, family, username, password = "en", "wikipedia", "user", "pass" api = ALL_APIS(lang, family, username, password) assert api.lang == lang @@ -35,47 +38,47 @@ def test_all_apis_init(mock_dependencies): assert api.password == password # Verify login was called - mock_dependencies['Login'].assert_called_once_with(lang, family=family) - mock_dependencies['LoginInstance'].add_users.assert_called_once() + mock_dependencies["Login"].assert_called_once_with(lang, family=family) + mock_dependencies["LoginInstance"].add_users.assert_called_once() # Verify printe.output was called - mock_dependencies['printe'].output.assert_called() + mock_dependencies["printe"].output.assert_called() def test_all_apis_main_page(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') - title = 'Test Page' + api = ALL_APIS("en", "wikipedia", "user", "pass") + title = "Test Page" api.MainPage(title) - mock_dependencies['MainPage'].assert_called_once_with( - mock_dependencies['LoginInstance'], title, 'en', family='wikipedia' + mock_dependencies["MainPage"].assert_called_once_with( + mock_dependencies["LoginInstance"], title, "en", family="wikipedia" ) def test_all_apis_cat_depth(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') - title = 'Category:Test' + api = ALL_APIS("en", "wikipedia", "user", "pass") + title = "Category:Test" api.CatDepth(title, depth=2) - mock_dependencies['subcatquery'].assert_called_once_with( - mock_dependencies['LoginInstance'], title, sitecode='en', family='wikipedia', depth=2 + mock_dependencies["subcatquery"].assert_called_once_with( + mock_dependencies["LoginInstance"], title, sitecode="en", family="wikipedia", depth=2 ) def test_all_apis_new_api(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') + api = ALL_APIS("en", "wikipedia", "user", "pass") api.NEW_API() - mock_dependencies['NEW_API'].assert_called_once_with( - mock_dependencies['LoginInstance'], lang='en', family='wikipedia' + mock_dependencies["NEW_API"].assert_called_once_with( + mock_dependencies["LoginInstance"], lang="en", family="wikipedia" ) def test_login_lru_cache(mock_dependencies): - api = ALL_APIS('en', 'wikipedia', 'user', 'pass') + api = ALL_APIS("en", "wikipedia", "user", "pass") # Call _login multiple times api._login() @@ -83,4 +86,4 @@ def test_login_lru_cache(mock_dependencies): # Login should only be instantiated once due to lru_cache # Note: __init__ calls _login once, so total calls should be 1 - assert mock_dependencies['Login'].call_count == 1 + assert mock_dependencies["Login"].call_count == 1 diff --git a/tests/TestAuthentication.py b/tests/TestAuthentication.py index fe47715..d527062 100644 --- a/tests/TestAuthentication.py +++ b/tests/TestAuthentication.py @@ -2,13 +2,11 @@ from newapi import useraccount from newapi.super import super_login + class TestAuthentication: @pytest.fixture def user_credentials(self): - return { - "username": useraccount.username, - "password": useraccount.password - } + return {"username": useraccount.username, "password": useraccount.password} @pytest.fixture def login_client(self, user_credentials): @@ -18,21 +16,14 @@ def login_client(self, user_credentials): def test_successful_login(self, login_client): """Test successful authentication""" - params = { - "action": "query", - "titles": "Main Page", - "format": "json" - } + params = {"action": "query", "titles": "Main Page", "format": "json"} response = login_client.post(params, Type="post", addtoken=False) assert response is not None assert len(response) > 0 def test_invalid_credentials(self): """Test authentication with invalid credentials""" - invalid_creds = { - "username": useraccount.username, - "password": f"{useraccount.password}213" - } + invalid_creds = {"username": useraccount.username, "password": f"{useraccount.password}213"} bot = super_login.Login("en", family="wikipedia") # Test should handle authentication failure gracefully login_result = bot.Log_to_wiki() diff --git a/tests/TestLiteDB.py b/tests/TestLiteDB.py index dd47f27..64657e1 100644 --- a/tests/TestLiteDB.py +++ b/tests/TestLiteDB.py @@ -1,13 +1,15 @@ -import pytest -import tempfile import os +import tempfile + +import pytest from newapi.DB_bots.db_bot import LiteDB + class TestLiteDB: @pytest.fixture def temp_db(self): """Create temporary database for testing""" - fd, path = tempfile.mkstemp(suffix='.db') + fd, path = tempfile.mkstemp(suffix=".db") os.close(fd) db = LiteDB(path) yield db diff --git a/tests/TestMainPage.py b/tests/TestMainPage.py index c7269ff..6c6cd8d 100644 --- a/tests/TestMainPage.py +++ b/tests/TestMainPage.py @@ -1,10 +1,11 @@ import pytest from newapi.page import MainPage + class TestMainPage: @pytest.fixture def test_page(self): - return MainPage("User:Mr. Ibrahem/sandbox", 'en') + return MainPage("User:Mr. Ibrahem/sandbox", "en") @pytest.fixture def arabic_page(self): @@ -29,7 +30,7 @@ def test_get_text(self, arabic_page): def test_nonexistent_page(self): """Test behavior with non-existent page""" - page = MainPage("NonExistentPage12345", 'en') + page = MainPage("NonExistentPage12345", "en") assert page.exists() is False assert isinstance(page.get_text(), str) # Should handle gracefully @@ -41,7 +42,7 @@ def test_empty_page_content(self): def test_page_without_edit_permission(self): """Test page where user cannot edit""" # Test with a protected page or mock this scenario - page = MainPage("الصفحة الرئيسة", 'ar') + page = MainPage("الصفحة الرئيسة", "ar") assert page.can_edit() is False def test_page_title_validation(self): diff --git a/tests/TestNewAPI.py b/tests/TestNewAPI.py index 6abeb5c..657b63b 100644 --- a/tests/TestNewAPI.py +++ b/tests/TestNewAPI.py @@ -9,10 +9,7 @@ def api_client(self): def test_find_pages_exists(self, api_client): """Test Find_pages_exists_or_not method""" - result = api_client.Find_pages_exists_or_not( - ["Thyrotropin alfa", "Thiamine"], - get_redirect=True - ) + result = api_client.Find_pages_exists_or_not(["Thyrotropin alfa", "Thiamine"], get_redirect=True) assert result is not None assert isinstance(result, (dict, list)) diff --git a/tests/conftest.py b/tests/conftest.py index 6c048e4..1b58457 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,19 @@ -import pytest -import sys import os +import sys + +import pytest # إضافة مسار المشروع إلى sys.path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + @pytest.fixture(scope="session") def api_credentials(): """بيانات اعتماد مشتركة للاختبارات""" from newapi import useraccount - return { - "username": useraccount.username, - "password": useraccount.password - } + + return {"username": useraccount.username, "password": useraccount.password} + @pytest.fixture def temp_test_page(): diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py index 0757eda..82f09d8 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates.py @@ -4,14 +4,15 @@ Tests cover all scenarios including template restrictions, caching behavior, and special template handling (nobots, bots). """ + import sys -import pytest +import pytest from newapi.api_utils.bot_edit.bot_edit_by_templates import ( - is_bot_edit_allowed, Bot_Cache, - stop_edit_temps, edit_username, + is_bot_edit_allowed, + stop_edit_temps, ) @@ -369,10 +370,6 @@ def test_different_botjob_not_affected_by_specific_template(self, original_argv) sys.argv = ["script"] text = "{{لا للتعريب}}" # Should block for 'تعريب' botjob - assert not is_bot_edit_allowed( - text=text, title_page="Test", botjob="تعريب" - ) + assert not is_bot_edit_allowed(text=text, title_page="Test", botjob="تعريب") # Should allow for 'all' botjob (not in 'all' stop list) - assert is_bot_edit_allowed( - text=text, title_page="Test2", botjob="all" - ) + assert is_bot_edit_allowed(text=text, title_page="Test2", botjob="all") diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py index b651959..1a12a11 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates2.py @@ -8,26 +8,26 @@ - Edge cases and special conditions """ -import pytest -from unittest.mock import patch, MagicMock import sys +from unittest.mock import MagicMock, patch +import pytest from newapi.api_utils.bot_edit.bot_edit_by_templates import ( - is_bot_edit_allowed, Bot_Cache, + is_bot_edit_allowed, stop_edit_temps, ) - # ==================== Fixtures ==================== + @pytest.fixture(autouse=True) def reset_environment(): """Reset environment before and after each test.""" # Setup Bot_Cache.clear() original_argv = sys.argv.copy() - sys.argv = ['test'] + sys.argv = ["test"] yield @@ -39,13 +39,14 @@ def reset_environment(): @pytest.fixture def mock_wtp(): """Provide a mocked wikitextparser.""" - with patch('newapi.api_utils.bot_edit.bot_edit_by_templates.wtp') as mock: + with patch("newapi.api_utils.bot_edit.bot_edit_by_templates.wtp") as mock: yield mock @pytest.fixture def create_mock_template(): """Factory fixture for creating mock templates.""" + def _create_template(name, arguments=None): mock_template = MagicMock() mock_template.normal_name.return_value = name @@ -70,12 +71,13 @@ def _create_template(name, arguments=None): @pytest.fixture def setup_parser(mock_wtp, create_mock_template): """Factory fixture for setting up parser with templates.""" + def _setup(templates_config): templates = [] for config in templates_config: if isinstance(config, dict): - name = config.get('name') - arguments = config.get('arguments') + name = config.get("name") + arguments = config.get("arguments") templates.append(create_mock_template(name, arguments)) else: # If just a string, create template with that name @@ -92,6 +94,7 @@ def _setup(templates_config): # ==================== Basic Functionality Tests ==================== + class TestBasicFunctionality: """Test basic functionality of is_bot_edit_allowed.""" @@ -116,17 +119,14 @@ def test_default_botjob_parameter(self): def test_fixref_cat_stub_tempcat_portal_defaults_to_all(self): """Test that combined botjob string defaults to 'all'.""" text = "Plain text" - result = is_bot_edit_allowed( - text=text, - title_page="Test Page", - botjob="fixref|cat|stub|tempcat|portal" - ) + result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="fixref|cat|stub|tempcat|portal") assert result is True assert "all" in Bot_Cache # ==================== Cache Tests ==================== + class TestCacheBehavior: """Test cache behavior and management.""" @@ -186,6 +186,7 @@ def test_cache_persists_across_calls(self): assert first_cache_value == second_cache_value + # ==================== Nobots Template Tests ==================== @@ -194,7 +195,7 @@ class TestNobotsTemplate: def test_nobots_without_params_denies_edit(self, setup_parser): """Test that {{nobots}} without parameters denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) text = "{{nobots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -202,7 +203,7 @@ def test_nobots_without_params_denies_edit(self, setup_parser): def test_nobots_with_all_denies_edit(self, setup_parser): """Test that {{nobots|1=all}} denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'all'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "all"}}]) text = "{{nobots|1=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -210,7 +211,7 @@ def test_nobots_with_all_denies_edit(self, setup_parser): def test_nobots_with_specific_bot_denies_edit(self, setup_parser): """Test that {{nobots|1=Mr.Ibrahembot}} denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'Mr.Ibrahembot'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "Mr.Ibrahembot"}}]) text = "{{nobots|1=Mr.Ibrahembot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -218,7 +219,7 @@ def test_nobots_with_specific_bot_denies_edit(self, setup_parser): def test_nobots_with_bot_list_including_our_bot_denies(self, setup_parser): """Test that {{nobots|1=Bot1,Mr.Ibrahembot,Bot2}} denies editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'Bot1,Mr.Ibrahembot,Bot2'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "Bot1,Mr.Ibrahembot,Bot2"}}]) text = "{{nobots|1=Bot1,Mr.Ibrahembot,Bot2}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -226,7 +227,7 @@ def test_nobots_with_bot_list_including_our_bot_denies(self, setup_parser): def test_nobots_with_other_bots_allows_edit(self, setup_parser): """Test that {{nobots|1=OtherBot}} allows editing.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'OtherBot,AnotherBot'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "OtherBot,AnotherBot"}}]) text = "{{nobots|1=OtherBot,AnotherBot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -234,7 +235,7 @@ def test_nobots_with_other_bots_allows_edit(self, setup_parser): def test_nobots_case_insensitive(self, setup_parser): """Test that nobots template matching is case insensitive.""" - setup_parser([{'name': 'NoBots', 'arguments': None}]) + setup_parser([{"name": "NoBots", "arguments": None}]) text = "{{NoBots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -242,7 +243,7 @@ def test_nobots_case_insensitive(self, setup_parser): def test_nobots_with_whitespace_in_bot_names(self, setup_parser): """Test handling of whitespace in bot name lists.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': ' Bot1 , Mr.Ibrahembot , Bot2 '}}]) + setup_parser([{"name": "nobots", "arguments": {"1": " Bot1 , Mr.Ibrahembot , Bot2 "}}]) text = "{{nobots|1= Bot1 , Mr.Ibrahembot , Bot2 }}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -251,12 +252,13 @@ def test_nobots_with_whitespace_in_bot_names(self, setup_parser): # ==================== Bots Template Tests ==================== + class TestBotsTemplate: """Test bots template handling.""" def test_bots_without_params_denies_edit(self, setup_parser): """Test that {{bots}} without parameters denies editing.""" - setup_parser([{'name': 'bots', 'arguments': None}]) + setup_parser([{"name": "bots", "arguments": None}]) text = "{{bots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -264,7 +266,7 @@ def test_bots_without_params_denies_edit(self, setup_parser): def test_bots_allow_all_allows_edit(self, setup_parser): """Test that {{bots|allow=all}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'all'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "all"}}]) text = "{{bots|allow=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -272,7 +274,7 @@ def test_bots_allow_all_allows_edit(self, setup_parser): def test_bots_allow_specific_bot_allows_edit(self, setup_parser): """Test that {{bots|allow=Mr.Ibrahembot}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'Mr.Ibrahembot'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "Mr.Ibrahembot"}}]) text = "{{bots|allow=Mr.Ibrahembot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -280,7 +282,7 @@ def test_bots_allow_specific_bot_allows_edit(self, setup_parser): def test_bots_allow_bot_list_including_our_bot_allows(self, setup_parser): """Test that {{bots|allow=Bot1,Mr.Ibrahembot,Bot2}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'Bot1,Mr.Ibrahembot,Bot2'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "Bot1,Mr.Ibrahembot,Bot2"}}]) text = "{{bots|allow=Bot1,Mr.Ibrahembot,Bot2}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -288,7 +290,7 @@ def test_bots_allow_bot_list_including_our_bot_allows(self, setup_parser): def test_bots_allow_none_denies_edit(self, setup_parser): """Test that {{bots|allow=none}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'none'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "none"}}]) text = "{{bots|allow=none}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -296,7 +298,7 @@ def test_bots_allow_none_denies_edit(self, setup_parser): def test_bots_allow_other_bots_denies_edit(self, setup_parser): """Test that {{bots|allow=OtherBot}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': 'OtherBot,AnotherBot'}}]) + setup_parser([{"name": "bots", "arguments": {"allow": "OtherBot,AnotherBot"}}]) text = "{{bots|allow=OtherBot,AnotherBot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -304,7 +306,7 @@ def test_bots_allow_other_bots_denies_edit(self, setup_parser): def test_bots_deny_all_denies_edit(self, setup_parser): """Test that {{bots|deny=all}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': 'all'}}]) + setup_parser([{"name": "bots", "arguments": {"deny": "all"}}]) text = "{{bots|deny=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -312,7 +314,7 @@ def test_bots_deny_all_denies_edit(self, setup_parser): def test_bots_deny_specific_bot_denies_edit(self, setup_parser): """Test that {{bots|deny=Mr.Ibrahembot}} denies editing.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': 'Mr.Ibrahembot'}}]) + setup_parser([{"name": "bots", "arguments": {"deny": "Mr.Ibrahembot"}}]) text = "{{bots|deny=Mr.Ibrahembot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -320,7 +322,7 @@ def test_bots_deny_specific_bot_denies_edit(self, setup_parser): def test_bots_deny_other_bots_allows_edit(self, setup_parser): """Test that {{bots|deny=OtherBot}} allows editing.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': 'OtherBot,AnotherBot'}}]) + setup_parser([{"name": "bots", "arguments": {"deny": "OtherBot,AnotherBot"}}]) text = "{{bots|deny=OtherBot,AnotherBot}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -328,7 +330,7 @@ def test_bots_deny_other_bots_allows_edit(self, setup_parser): def test_bots_case_insensitive(self, setup_parser): """Test that bots template matching is case insensitive.""" - setup_parser([{'name': 'Bots', 'arguments': {'allow': 'all'}}]) + setup_parser([{"name": "Bots", "arguments": {"allow": "all"}}]) text = "{{Bots|allow=all}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -337,13 +339,14 @@ def test_bots_case_insensitive(self, setup_parser): # ==================== Stop Edit Templates Tests ==================== + class TestStopEditTemplates: """Test stop edit templates handling.""" @pytest.mark.parametrize("template_name", stop_edit_temps["all"]) def test_global_stop_templates_deny_edit(self, template_name, setup_parser): """Test that global stop templates deny editing.""" - setup_parser([{'name': template_name, 'arguments': None}]) + setup_parser([{"name": template_name, "arguments": None}]) text = f"{{{{{template_name}}}}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -352,7 +355,7 @@ def test_global_stop_templates_deny_edit(self, template_name, setup_parser): def test_specific_botjob_stop_template_denies_edit(self, setup_parser): """Test that job-specific stop templates deny editing.""" # Test تعريب job with its specific template - setup_parser([{'name': 'لا للتعريب', 'arguments': None}]) + setup_parser([{"name": "لا للتعريب", "arguments": None}]) text = "{{لا للتعريب}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="تعريب") @@ -361,22 +364,20 @@ def test_specific_botjob_stop_template_denies_edit(self, setup_parser): def test_stop_template_for_different_botjob_allows_edit(self, setup_parser): """Test that stop templates for different bot jobs allow editing.""" # Template for تعريب job, but we're running cat job - setup_parser([{'name': 'لا للتعريب', 'arguments': None}]) + setup_parser([{"name": "لا للتعريب", "arguments": None}]) text = "{{لا للتعريب}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="cat") assert result is True - @pytest.mark.parametrize("botjob,template_list", [ - (job, templates) - for job, templates in stop_edit_temps.items() - if job != "all" - ]) + @pytest.mark.parametrize( + "botjob,template_list", [(job, templates) for job, templates in stop_edit_temps.items() if job != "all"] + ) def test_all_stop_templates_for_each_botjob(self, botjob, template_list, setup_parser): """Test all stop templates for each specific bot job.""" for template_name in template_list: Bot_Cache.clear() - setup_parser([{'name': template_name, 'arguments': None}]) + setup_parser([{"name": template_name, "arguments": None}]) text = f"{{{{{template_name}}}}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob=botjob) @@ -385,15 +386,13 @@ def test_all_stop_templates_for_each_botjob(self, botjob, template_list, setup_p # ==================== Multiple Templates Tests ==================== + class TestMultipleTemplates: """Test handling of multiple templates.""" def test_multiple_templates_first_restricting_denies(self, setup_parser): """Test that first restricting template denies editing.""" - setup_parser([ - {'name': 'nobots', 'arguments': None}, - {'name': 'some_other_template', 'arguments': None} - ]) + setup_parser([{"name": "nobots", "arguments": None}, {"name": "some_other_template", "arguments": None}]) text = "{{nobots}} {{some_other_template}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -401,10 +400,7 @@ def test_multiple_templates_first_restricting_denies(self, setup_parser): def test_multiple_non_restricting_templates_allows(self, setup_parser): """Test that multiple non-restricting templates allow editing.""" - setup_parser([ - {'name': 'infobox', 'arguments': None}, - {'name': 'citation', 'arguments': None} - ]) + setup_parser([{"name": "infobox", "arguments": None}, {"name": "citation", "arguments": None}]) text = "{{infobox}} {{citation}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -413,12 +409,13 @@ def test_multiple_non_restricting_templates_allows(self, setup_parser): # ==================== Edge Cases Tests ==================== + class TestEdgeCases: """Test edge cases and special conditions.""" def test_empty_template_parameters(self, setup_parser): """Test handling of templates with empty parameter values.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': ''}}]) + setup_parser([{"name": "nobots", "arguments": {"1": ""}}]) text = "{{nobots|1=}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -427,10 +424,7 @@ def test_empty_template_parameters(self, setup_parser): def test_template_with_multiple_parameters(self, setup_parser): """Test template with multiple parameters.""" - setup_parser([{ - 'name': 'bots', - 'arguments': {'allow': 'all', 'other_param': 'some_value'} - }]) + setup_parser([{"name": "bots", "arguments": {"allow": "all", "other_param": "some_value"}}]) text = "{{bots|allow=all|other_param=some_value}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -438,10 +432,7 @@ def test_template_with_multiple_parameters(self, setup_parser): def test_parameter_filtering_empty_values(self, setup_parser): """Test that parameters with empty values are filtered out.""" - setup_parser([{ - 'name': 'bots', - 'arguments': {'allow': 'all', 'empty_param': ''} - }]) + setup_parser([{"name": "bots", "arguments": {"allow": "all", "empty_param": ""}}]) text = "{{bots|allow=all|empty_param=}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") @@ -450,14 +441,18 @@ def test_parameter_filtering_empty_values(self, setup_parser): # ==================== Parametrized Test Collections ==================== + class TestParametrizedScenarios: """Parametrized tests for various scenarios.""" - @pytest.mark.parametrize("text,expected", [ - ("", True), - ("Plain text", True), - ("Some content without templates", True), - ]) + @pytest.mark.parametrize( + "text,expected", + [ + ("", True), + ("Plain text", True), + ("Some content without templates", True), + ], + ) def test_non_template_text_allows_editing(self, text, expected): """Test that non-template text allows editing.""" result = is_bot_edit_allowed(text=text, title_page="Test", botjob="all") @@ -470,44 +465,53 @@ def test_cache_initialized_for_different_botjobs(self, botjob): expected_job = "all" if botjob in ["", "fixref|cat|stub|tempcat|portal"] else botjob assert expected_job in Bot_Cache - @pytest.mark.parametrize("bot_list,should_allow", [ - ("OtherBot", True), - ("Bot1,Bot2,Bot3", True), - ("Mr.Ibrahembot", False), - ("Bot1,Mr.Ibrahembot", False), - ("all", False), - ]) + @pytest.mark.parametrize( + "bot_list,should_allow", + [ + ("OtherBot", True), + ("Bot1,Bot2,Bot3", True), + ("Mr.Ibrahembot", False), + ("Bot1,Mr.Ibrahembot", False), + ("all", False), + ], + ) def test_nobots_with_various_bot_lists(self, bot_list, should_allow, setup_parser): """Test nobots template with various bot lists.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': bot_list}}]) + setup_parser([{"name": "nobots", "arguments": {"1": bot_list}}]) result = is_bot_edit_allowed(text="{{nobots}}", title_page="Test", botjob="all") assert result == should_allow - @pytest.mark.parametrize("allow_list,should_allow", [ - ("all", True), - ("Mr.Ibrahembot", True), - ("Bot1,Mr.Ibrahembot,Bot2", True), - ("none", False), - ("OtherBot", False), - ]) + @pytest.mark.parametrize( + "allow_list,should_allow", + [ + ("all", True), + ("Mr.Ibrahembot", True), + ("Bot1,Mr.Ibrahembot,Bot2", True), + ("none", False), + ("OtherBot", False), + ], + ) def test_bots_allow_with_various_lists(self, allow_list, should_allow, setup_parser): """Test bots allow parameter with various lists.""" - setup_parser([{'name': 'bots', 'arguments': {'allow': allow_list}}]) + setup_parser([{"name": "bots", "arguments": {"allow": allow_list}}]) result = is_bot_edit_allowed(text="{{bots}}", title_page="Test", botjob="all") assert result == should_allow - @pytest.mark.parametrize("deny_list,should_allow", [ - ("all", False), - ("Mr.Ibrahembot", False), - ("Bot1,Mr.Ibrahembot,Bot2", False), - ("OtherBot", True), - ("Bot1,Bot2", True), - ]) + @pytest.mark.parametrize( + "deny_list,should_allow", + [ + ("all", False), + ("Mr.Ibrahembot", False), + ("Bot1,Mr.Ibrahembot,Bot2", False), + ("OtherBot", True), + ("Bot1,Bot2", True), + ], + ) def test_bots_deny_with_various_lists(self, deny_list, should_allow, setup_parser): """Test bots deny parameter with various lists.""" - setup_parser([{'name': 'bots', 'arguments': {'deny': deny_list}}]) + setup_parser([{"name": "bots", "arguments": {"deny": deny_list}}]) result = is_bot_edit_allowed(text="{{bots}}", title_page="Test", botjob="all") assert result == should_allow @@ -515,27 +519,20 @@ def test_bots_deny_with_various_lists(self, deny_list, should_allow, setup_parse # ==================== Integration Tests ==================== + class TestIntegration: """Integration tests for complete workflows.""" def test_complete_workflow_allowed(self, setup_parser): """Test complete workflow where editing is allowed.""" # Setup non-restricting template - setup_parser([{'name': 'infobox', 'arguments': None}]) + setup_parser([{"name": "infobox", "arguments": None}]) # First call - process and cache - result1 = is_bot_edit_allowed( - text="{{infobox}}", - title_page="Article", - botjob="all" - ) + result1 = is_bot_edit_allowed(text="{{infobox}}", title_page="Article", botjob="all") # Second call - use cache - result2 = is_bot_edit_allowed( - text="{{infobox}}", - title_page="Article", - botjob="all" - ) + result2 = is_bot_edit_allowed(text="{{infobox}}", title_page="Article", botjob="all") assert result1 is True assert result2 is True @@ -545,21 +542,13 @@ def test_complete_workflow_allowed(self, setup_parser): def test_complete_workflow_denied(self, setup_parser): """Test complete workflow where editing is denied.""" # Setup restricting template - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) # First call - process and cache - result1 = is_bot_edit_allowed( - text="{{nobots}}", - title_page="Article", - botjob="all" - ) + result1 = is_bot_edit_allowed(text="{{nobots}}", title_page="Article", botjob="all") # Second call - use cache - result2 = is_bot_edit_allowed( - text="{{nobots}}", - title_page="Article", - botjob="all" - ) + result2 = is_bot_edit_allowed(text="{{nobots}}", title_page="Article", botjob="all") assert result1 is False assert result2 is False @@ -569,11 +558,11 @@ def test_complete_workflow_denied(self, setup_parser): def test_different_pages_different_results(self, setup_parser): """Test that different pages can have different results.""" # Page 1 - allowed - setup_parser([{'name': 'infobox', 'arguments': None}]) + setup_parser([{"name": "infobox", "arguments": None}]) result1 = is_bot_edit_allowed(text="{{infobox}}", title_page="Page1", botjob="all") # Page 2 - denied - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) result2 = is_bot_edit_allowed(text="{{nobots}}", title_page="Page2", botjob="all") assert result1 is True @@ -584,29 +573,30 @@ def test_different_pages_different_results(self, setup_parser): # ==================== Performance and Special Cases ==================== + class TestSpecialCases: """Test special cases and boundary conditions.""" def test_very_long_bot_list(self, setup_parser): """Test handling of very long bot lists.""" long_list = ",".join([f"Bot{i}" for i in range(100)]) - setup_parser([{'name': 'nobots', 'arguments': {'1': long_list}}]) + setup_parser([{"name": "nobots", "arguments": {"1": long_list}}]) result = is_bot_edit_allowed(text="{{nobots}}", title_page="Test", botjob="all") assert result is True # Our bot not in the list def test_unicode_bot_names(self, setup_parser): """Test handling of unicode characters in bot names.""" - setup_parser([{'name': 'nobots', 'arguments': {'1': 'بوت1,بوت2'}}]) + setup_parser([{"name": "nobots", "arguments": {"1": "بوت1,بوت2"}}]) result = is_bot_edit_allowed(text="{{nobots}}", title_page="Test", botjob="all") assert result is True # Our bot not in the list def test_mixed_case_template_names(self, setup_parser): """Test that template name matching works with mixed case.""" - for name in ['nobots', 'Nobots', 'NOBOTS', 'nObOtS']: + for name in ["nobots", "Nobots", "NOBOTS", "nObOtS"]: Bot_Cache.clear() - setup_parser([{'name': name, 'arguments': None}]) + setup_parser([{"name": name, "arguments": None}]) result = is_bot_edit_allowed(text=f"{{{{{name}}}}}", title_page="Test", botjob="all") assert result is False, f"Should deny for template name: {name}" @@ -614,6 +604,7 @@ def test_mixed_case_template_names(self, setup_parser): # ==================== Pytest Marks and Markers ==================== + class TestSmokeTests: """Quick smoke tests for CI/CD.""" @@ -623,7 +614,7 @@ def test_basic_allow(self): def test_basic_deny(self, setup_parser): """Basic deny scenario.""" - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) assert is_bot_edit_allowed("{{nobots}}", "Test", "all") is False def test_cache_works(self): diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py index f964873..48e5cb3 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_templates/test_bot_edit_by_templates_pypass.py @@ -1,13 +1,12 @@ -""" -""" +""" """ -import pytest import sys -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch +import pytest from newapi.api_utils.bot_edit.bot_edit_by_templates import ( - is_bot_edit_allowed, Bot_Cache, + is_bot_edit_allowed, ) # ==================== Fixtures ==================== @@ -27,7 +26,7 @@ def reset_environment(): # Setup Bot_Cache.clear() original_argv = sys.argv.copy() - sys.argv = ['test'] + sys.argv = ["test"] yield @@ -39,13 +38,14 @@ def reset_environment(): @pytest.fixture def mock_wtp(): """Provide a mocked wikitextparser.""" - with patch('newapi.api_utils.bot_edit.bot_edit_by_templates.wtp') as mock: + with patch("newapi.api_utils.bot_edit.bot_edit_by_templates.wtp") as mock: yield mock @pytest.fixture def create_mock_template(): """Factory fixture for creating mock templates.""" + def _create_template(name, arguments=None): mock_template = MagicMock() mock_template.normal_name.return_value = name @@ -70,12 +70,13 @@ def _create_template(name, arguments=None): @pytest.fixture def setup_parser(mock_wtp, create_mock_template): """Factory fixture for setting up parser with templates.""" + def _setup(templates_config): templates = [] for config in templates_config: if isinstance(config, dict): - name = config.get('name') - arguments = config.get('arguments') + name = config.get("name") + arguments = config.get("arguments") templates.append(create_mock_template(name, arguments)) else: # If just a string, create template with that name @@ -92,6 +93,7 @@ def _setup(templates_config): # ==================== Command Line Bypass Tests ==================== + class TestCommandLineBypass: """Test command line argument bypass functionality.""" @@ -99,7 +101,7 @@ class TestCommandLineBypass: def test_argv_bypasses_all_checks(self, argv_value, setup_parser): """Test that specific argv values bypass all restrictions.""" sys.argv.append(argv_value) - setup_parser([{'name': 'nobots', 'arguments': None}]) + setup_parser([{"name": "nobots", "arguments": None}]) text = "{{nobots}}" result = is_bot_edit_allowed(text=text, title_page="Test Page", botjob="all") diff --git a/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py b/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py index 2464ca5..594d828 100644 --- a/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py +++ b/tests/unit/api_utils/bot_edit/bot_edit_by_time/test_bot_edit_by_time.py @@ -1,8 +1,8 @@ -""" -""" +""" """ + import sys -import pytest +import pytest from newapi.api_utils.bot_edit.bot_edit_by_time import ( check_create_time, check_last_edit_time, diff --git a/z_te_sts/__init__.py b/z_te_sts/__init__.py deleted file mode 100644 index 40a96af..0000000 --- a/z_te_sts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/z_te_sts/nc_files.db b/z_te_sts/nc_files.db deleted file mode 100644 index 987fe97..0000000 Binary files a/z_te_sts/nc_files.db and /dev/null differ diff --git a/z_te_sts/test_bot_api.py b/z_te_sts/test_bot_api.py deleted file mode 100644 index efb50e6..0000000 --- a/z_te_sts/test_bot_api.py +++ /dev/null @@ -1,254 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys -import time - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.page import NEW_API -from newapi import printe - -class testmybot: - def __init__(self): - self.test = "test" - self.api_new = NEW_API("en", family="wikipedia") - # self.api_new.Login_to_wiki() - - def test1(self): - """Find_pages_exists_or_not""" - ex = self.api_new.Find_pages_exists_or_not(["Thyrotropin alfa", "Thiamine"], get_redirect=True) - return ex - - def test2(self): - """Get_All_pages""" - ex = self.api_new.Get_All_pages(start="!", limit_all=1000) - return ex - - def test3(self): - """Search""" - ex = self.api_new.Search("yemen", srlimit="1000") - return ex - - def test4(self): - """Get_Newpages""" - ex = self.api_new.Get_Newpages(limit=100) - return ex - - def test5(self): - """Get language links for a list of names. - - This method retrieves language links for a predefined list of names - using the `Get_langlinks_for_list` API method. The list contains various - individuals, and the function specifically targets the German language - site code ("de"). The results are returned for further processing or - validation. - - Returns: - list: A list of language links corresponding to the input names. - """ - lista = [ - "Amanda Seales", - "Aria Mia Loberti", - "Rachel Sennott", - "Bassam Tariq", - "Darius Marder", - "Elvira Lind", - "Shaka King", - "Dev Hynes", - "Michael Stipe", - "Raveena Aurora", - "Tommy Genesis", - "Vic Mensa", - "Becca Balint", - "Earl Blumenauer", - "Sanford Bishop", - "Suzanne Bonamici", - "Greg Casar", - "Jason Crow", - "Diana DeGette", - "Chris Deluzio", - "Mark DeSaulnier", - "Debbie Dingell", - "Veronica Escobar", - "Valerie Foushee", - "John Garamendi", - "Chuy García", - "Raúl Grijalva", - "Josh Harder", - "Chrissy Houlahan", - "Dan Kildee", - "Greg Landsman", - "Summer Lee", - "Teresa Leger Fernandez", - "Zoe Lofgren", - "Seth Magaziner", - "Betty McCollum", - "Morgan McGarvey", - "Kweisi Mfume", - "Donald Payne Jr.", - "Chellie Pingree", - "Pat Ryan (politician)", - "Jamie Raskin", - "Delia Ramirez", - "Mary Gay Scanlon", - "Jan Schakowsky", - "Terri Sewell", - "Brad Sherman", - "Mikie Sherrill", - "Paul Tonko", - "Lauren Underwood", - "Gabe Vasquez", - "Nydia Velázquez", - "Bonnie Watson Coleman", - ] - ex = self.api_new.Get_langlinks_for_list(lista, targtsitecode="de") - return ex - - def test6(self): - """UserContribs""" - ex = self.api_new.UserContribs("User:Mr. Ibahem", limit="10", namespace="*", ucshow="") - return ex - - def test7(self): - """expandtemplates""" - ex = self.api_new.expandtemplates("{{refn|Wing drop is an unc maneuvering.}}") - return ex - - def test8(self): - """get_extlinks""" - api_new = NEW_API("ar", family="wikipedia") - ex = api_new.get_extlinks("اليمن") - return ex - - def test9(self): - """querypage_list""" - ex = self.api_new.querypage_list(qppage="Wantedcategories", qplimit="max", Max=500) - return ex - - def test10(self): - """Get_template_pages""" - api_new = NEW_API("ar", family="wikipedia") - ex = api_new.Get_template_pages("قالب:طواف العالم للدراجات", namespace="*", Max=10000) - return ex - - def test11(self): - """move""" - api_new = NEW_API("ar", family="wikipedia") - move_it = api_new.move("الصفحة_الرئيسة", "الصفحة_الرئيسة2", reason="test!", noredirect=False, movesubpages=False) - return move_it - - def test12(self): - """Add_To_Bottom""" - api_new = NEW_API("ar", family="wikipedia") - move_it = api_new.Add_To_Bottom("x", "x", "الصفحة_الرئيسة") - return move_it - - def test13(self): - """get_pageassessments""" - move_it = self.api_new.get_pageassessments("yemen") - return move_it - - def test14(self): - """users_infos""" - api_new = NEW_API("ar", family="wikipedia") - users = api_new.users_infos(ususers=["Mr. Ibrahem", "93.112.145.86"]) - - return users - - def start(self): - """Start the execution of defined test functions based on command-line - arguments. - - This method initializes a dictionary of test functions and checks for - command-line arguments to determine which tests to execute. It processes - the arguments to filter the tests that should be run, executes them, and - outputs the results. If a test function returns a dictionary, it formats - and prints the results accordingly. Additionally, it handles specific - command-line flags to control the output. - - Raises: - Exception: If the result of a test function is an empty string. - """ - - # --- - defs1 = {} - # --- - defs = { - 1: self.test1, - 2: self.test2, - 3: self.test3, - 4: self.test4, - 5: self.test5, - 6: self.test6, - 7: self.test7, - 8: self.test8, - 9: self.test9, - 10: self.test10, - 11: self.test11, - 12: self.test12, - 13: self.test13, - 14: self.test14 - } - # --- - for arg in sys.argv: - arg, _, value = arg.partition(":") - if arg == "test" and value.isdigit(): - d = int(value) - defs1[d] = defs[d] - # --- - if defs1 != {}: - defs = defs1 - # --- - for n, func in defs.items(): - name = func.__name__ - printe.output(f"<> start def number {n}, name:{name}:", p=True) - # --- - def_name = func.__doc__ - printe.output(f"<> test: {def_name}:", p=True) - # --- - if "tat" in sys.argv: - continue - # --- - result = func() - # --- - # printe.output( result ) - # --- - if isinstance(result, dict): - for n, (na, ta) in enumerate(result.items()): - na2 = na # f" '{na}' ".ljust(10) - # --- - if na == "claims": - for x, u in ta.items(): - ta = {x: u} - break - # --- - # ta = json.dumps(ta, indent=2, ensure_ascii=False) - # --- - printe.output(f"{n}: {na2}: {ta}") - # --- - if result == "": - raise Exception("result == ''") - # --- - if "printresult" in sys.argv: - if isinstance(result, str): - printe.output(f"result:{result}") - elif isinstance(result, list): - printe.output(result) - else: - printe.output(result) - else: - printe.output("<> add 'printresult' to sys.argv to print result") - # --- - if result: - printe.output(f"{len(result)=}", p=True) - # --- - printe.output("=====================") - printe.output(f"<> test: {def_name} end...") - printe.output("time.sleep(1)") - time.sleep(1) - - -if __name__ == "__main__": - testmybot().start() diff --git a/z_te_sts/test_bot_edit.py b/z_te_sts/test_bot_edit.py deleted file mode 100644 index aad5a38..0000000 --- a/z_te_sts/test_bot_edit.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -from newapi.page import MainPage - -# --- -titles = [ - "أضرحة عائلة ميرزا أديغوزال بك", - "موسى والراعي (قصة)", - "جبل عمر", - "ويكيبيديا:ملعب", - "القدس خلال فترة الهيكل الثاني", - "أضرحة عائلة ميرزا أديغوزال بك", -] -# --- -for x in titles: - page = MainPage(x, "ar") - - canedit = page.can_edit(delay=30) - - # print(f"Page: {x}, \t:{canedit=}") - # break diff --git a/z_te_sts/test_catdepth.py b/z_te_sts/test_catdepth.py deleted file mode 100644 index a2dc9db..0000000 --- a/z_te_sts/test_catdepth.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys -# sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.page import CatDepth - -cat_members = CatDepth("اليمن", sitecode='ar', family="wikipedia", depth=0, ns="all", nslist=[], tempyes=[]) - -# print(cat_members.keys()) -print(f"xxxxxxxxxxxxxx: {len(cat_members)=}") - -cat_members2 = CatDepth("Yemen", sitecode='en', family="wikipedia", depth=0, ns="all", nslist=[], tempyes=[]) - -# print(cat_members2.keys()) -print(f"xxxxxxxxxxxxxx: {len(cat_members2)=}") - -cat_members3 = CatDepth("صنعاء", sitecode='ar', family="wikipedia", depth=0, ns="all", nslist=[], tempyes=[]) - -# print(cat_members3.keys()) -print(f"xxxxxxxxxxxxxx: {len(cat_members3)=}") diff --git a/z_te_sts/test_db_bot.py b/z_te_sts/test_db_bot.py deleted file mode 100644 index 0c79800..0000000 --- a/z_te_sts/test_db_bot.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import os -import sys -from datetime import datetime -from pathlib import Path -Dir = Path(__file__).resolve().parent - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi import db_bot - -LiteDB = db_bot.LiteDB - -def test(): - db_path = Dir / "nc_files.db" - if not os.path.exists(db_path): - print(f"Creating {db_path}") - db_path.touch() - - nc_files_db = LiteDB(db_path) - - nc_files_db.create_table( - "nc_files", - {"id": int, "lang": str, "title": str, "views": int, "dat": str}, - pk="id", - defaults={"views": 0}, - ) - - nc_files_db.show_tables() - - # Insert sample data - nc_files_db.insert( - "nc_files", - { - "lang": "English", - "title": "Sample Title 1", - # "views": 100, - "dat": datetime.now().strftime("%Y-%m-%d"), - }, - ) - - nc_files_db.insert( - "nc_files", - { - "lang": "French", - "title": "Sample Title 2", - # "views": 200, - "dat": datetime.now().strftime("%Y-%m-%d"), - }, - ) - - # Retrieve data - data = nc_files_db.get_data("nc_files") - for row in data: - print(row) - - -def test2(): - db_path = "/data/mdwiki/public_html/ncc/Tables/nc_files.db" - if not os.path.exists(db_path): - db_path = "I:/mdwiki/pybot/ncc_core/nc_import/bots/nc_files.db" - - nc_files_db = LiteDB(db_path) - - print("________") - # Select data - # data = nc_files_db.select("nc_files", {"lang": "English"}) - # print(data) - # for row in data: - # print(row) - - # Retrieve data - data = nc_files_db.get_data("nc_files") - for row in data: - print(row) - - -if __name__ == "__main__": - test() - test2() diff --git a/z_te_sts/test_langs.py b/z_te_sts/test_langs.py deleted file mode 100644 index edaf4da..0000000 --- a/z_te_sts/test_langs.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.page import MainPage - -page = MainPage('وب:ملعب', 'ar') -exists = page.exists() - -# --- -print('--------------') -print('simple:') -from newapi.page import NEW_API - -# --- -en_api_new = NEW_API('simple', family='wikipedia') -# --- -# en_api_new.Login_to_wiki() -# --- -pages = en_api_new.Find_pages_exists_or_not(['yemen']) -# --- -print('--------------') -save_page = page.save(newtext='test!', summary='', nocreate=1, minor='') diff --git a/z_te_sts/test_login.py b/z_te_sts/test_login.py deleted file mode 100644 index 7fd9114..0000000 --- a/z_te_sts/test_login.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.accounts.useraccount import User_tables_bot -from newapi.super import super_login - -User_tables = User_tables_bot -# --- - -Login = super_login.Login -# --- -bot = Login("ar", family="wikipedia") -# --- -bot.add_User_tables("wikipedia", User_tables) -# --- -params = {"action": "query", "titles": f"User:{User_tables['username']}", "prop": "revisions", "rvprop": "content", "rvslots": "*", "format": "json"} -# --- -json1 = bot.post(params, Type="post", addtoken=False) -# --- -# print(json1) - -print(f"{len(json1)=}") diff --git a/z_te_sts/test_login1.py b/z_te_sts/test_login1.py deleted file mode 100644 index 3f92dea..0000000 --- a/z_te_sts/test_login1.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.super import super_login - -from newapi.accounts.useraccount import User_tables_bot - -User_tables = User_tables_bot -User_tables["password"] += "123" -# --- - -# super_login.User_tables["wikipedia"] = User_tables - -Login = super_login.Login -# --- -bot = Login("simple", family="wikipedia") -login = bot.Log_to_wiki() -# --- -params = {"action": "query", "titles": "User:Mr. Ibrahem", "prop": "revisions", "rvprop": "content", "rvslots": "*", "format": "json"} -# --- -json1 = bot.post(params, Type="post", addtoken=False) -# --- -print(json1) - -print(f"{len(json1)=}") -# --- diff --git a/z_te_sts/test_ncc_page.py b/z_te_sts/test_ncc_page.py deleted file mode 100644 index 62cf840..0000000 --- a/z_te_sts/test_ncc_page.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.ncc_page import MainPage, CatDepth - -title = "Category:Pages_with_script_errors" - -# cat_members = CatDepth(title, depth=0, ns="10", nslist=[], tempyes=[]) - -cat_members = CatDepth(title, sitecode='www', family="nccommons", depth=0, onlyns=10) - -# print(cat_members) -print(f"{len(cat_members)=}") - -page = MainPage("Bilateral mesial temporal polymicrogyria (Radiopaedia 76456-88181 Axial SWI)", "www", family="nccommons") -# --- -text = page.get_text() -print(f"{len(text)=}") diff --git a/z_te_sts/test_page.py b/z_te_sts/test_page.py deleted file mode 100644 index ec151dc..0000000 --- a/z_te_sts/test_page.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys -# sys.argv.append("printurl") -sys.argv.append("ask") -from newapi.page import MainPage - -page = MainPage("وبxx:ملعب", "ar") -print(page.exists()) - -pageen = MainPage("الصفحة الرxئيسة", 'ar') -print(pageen.exists()) - - -# --- -page = MainPage("وب:ملعب", "ar") -pageen = MainPage("User:Mr. Ibrahem/sandbox", 'simple') -# --- -existsen = pageen.exists() - -if not page.can_edit(): - print("page can't edit!") -# --- -""" -if page.isRedirect() : return -# target = page.get_redirect_target() -# --- -text = page.get_text() -ns = page.namespace() -links = page.page_links() -categories = page.get_categories(with_hidden=False) -langlinks = page.get_langlinks() -back_links = page.page_backlinks() -wiki_links = page.get_wiki_links_from_text() -words = page.get_words() -templates = page.get_templates() -save_page = page.save(newtext='', summary='', nocreate=1, minor='') -create = page.Create(text='', summary='') -# --- -text_html = page.get_text_html() -hidden_categories= page.get_hidden_categories() -flagged = page.is_flagged() -timestamp = page.get_timestamp() -user = page.get_user() -purge = page.purge() -""" - -# --- -text = page.get_text() -print("---------------------------") -print(f"text:{len(text)=}") -# --- -ex = page.get_wiki_links_from_text() -print("---------------------------") -print(f"get_wiki_links_from_text:{len(ex)=}") -# --- -hidden_categories = page.get_hidden_categories() -print("---------------------------") -print(f"hidden_categories:{len(hidden_categories)=}") -# --- -backlinks = page.page_backlinks() -print("---------------------------") -print(f"backlinks:{len(backlinks)=}") -# --- - -newtext = "تجربة!\n" * 6 -# --- -save = page.save(newtext=newtext) - - -pageen.save(newtext="!!!", nocreate=0) - - -save = page.save(newtext="!") - diff --git a/z_te_sts/test_runner.py b/z_te_sts/test_runner.py deleted file mode 100644 index 5feb561..0000000 --- a/z_te_sts/test_runner.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -""" -Test runner for all z_te_sts/*.py files -Usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" - -import io -from contextlib import redirect_stdout -import sys -import os -import importlib.util -import traceback -from pathlib import Path - -# Add current directory to path for imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - - -def run_test_file(file_path): - """Run a single test file""" - try: - print(f"\n{'='*60}") - print(f"Running: {file_path.name}") - print(f"{'='*60}") - - # Load the module - spec = importlib.util.spec_from_file_location(file_path.stem, file_path) - module = importlib.util.module_from_spec(spec) - - # Capture stdout - - f = io.StringIO() - with redirect_stdout(f): - spec.loader.exec_module(module) - - output = f.getvalue() - if output: - print("Output:") - print(output) - - print(f"✓ {file_path.name} completed successfully") - return True - - except Exception as e: - print(f"✗ {file_path.name} failed: {e}") - print(traceback.format_exc()) - return False - - -def main(): - """Run all test files""" - # Get directory of this script - test_dir = Path(__file__).parent - - # Find all test files - test_files = [] - for file_path in test_dir.glob("test_*.py"): - if file_path.name != "test_runner.py": # Skip this file - test_files.append(file_path) - - if not test_files: - print("No test files found in z_te_sts directory") - return - - # Sort files for consistent execution - test_files.sort() - - print(f"Found {len(test_files)} test files:") - for file_path in test_files: - print(f" - {file_path.name}") - - print("\nStarting test execution...") - - # Run all tests - passed = 0 - failed = 0 - - for file_path in test_files: - if run_test_file(file_path): - passed += 1 - else: - failed += 1 - - # Summary - print(f"\n{'='*60}") - print("Test Summary") - print(f"{'='*60}") - print(f"Total tests: {len(test_files)}") - print(f"Passed: {passed}") - print(f"Failed: {failed}") - - if failed > 0: - print(f"\n❌ {failed} test(s) failed") - sys.exit(1) - else: - print("\n✅ All tests passed!") - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/z_te_sts/test_toolforge_page.py b/z_te_sts/test_toolforge_page.py deleted file mode 100644 index 67dad18..0000000 --- a/z_te_sts/test_toolforge_page.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.path.append("I:/mdwiki/pybot") -sys.argv.append("ask") -from newapi import toolforge_page - -from copy_to_en.bots import medwiki_account - -User_tables_md = { - "username": medwiki_account.username, - "password": medwiki_account.password, -} -# --- -User_tables_mdcx = { - "username": medwiki_account.username_cx, - "password": medwiki_account.password_cx, -} -# --- -toolforge_page.add_User_table(User_tables_md, "toolforge", "medwiki") -# --- -toolforge_page.add_User_table(User_tables_mdcx, "toolforge", "mdwikicx") -# --- -CatDepth = toolforge_page.CatDepth -MainPage = toolforge_page.MainPage - -lists = { - "page" : "Main_Page", - "cat" : "Category:Translations" -} -# --- -for site in ["medwiki", "mdwikicx"]: - # --- - print(f"___________________{site=}___________________") - # --- - print("//////// page ////////") - # --- - title = "Main_Page" - Category = "Category:Translations" - # --- - print(f"{site=}, {title=}") - # --- - page = MainPage(title, site, family="toolforge") - # --- - text = page.get_text() - print(f"{len(text)=}") - # --- - print("//////// Category ////////") - # --- - members = CatDepth(Category, sitecode=site, family="toolforge", ns=14) - # --- - print(members.keys()) - print(f"{site=} {Category=}, members: {len(members)}") diff --git a/z_te_sts/test_wiki_page.py b/z_te_sts/test_wiki_page.py deleted file mode 100644 index 8e2ffad..0000000 --- a/z_te_sts/test_wiki_page.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Test runner usage: python3 core8/pwb.py newapi_bot/z_te_sts/test_runner -""" -import sys - -sys.argv.append("printurl") -sys.argv.append("ask") - -from newapi.page import MainPage, CatDepth - -""" -page = MainPage(title, 'ar', family='wikipedia') -exists = page.exists() -text = page.get_text() -timestamp = page.get_timestamp() -user = page.get_user() -links = page.page_links() -words = page.get_words() -purge = page.purge() -templates = page.get_templates() -save_page = page.save(newtext='', summary='', nocreate=1, minor='') -create = page.Create(text='', summary='') -""" -# --- -page = MainPage("تصنيف:اليمن", "ar", family="wikipedia") -# --- -text = page.get_text() -print(f"{len(text)=}") - -# --- -# ex = page.page_backlinks() -# print('---------------------------') -# print(f'page_backlinks:{ex}') -page2 = MainPage("Category:Yemen", "en", family="wikipedia") -# --- -text2 = page2.get_text() -print(f"{len(text2)=}") -# --- -page_backlinks = page.page_backlinks() -print("---------------------------") -print(f"{len(page_backlinks)=}") - -# --- -# --- -# hidden_categories= page.get_hidden_categories() -# print('---------------------------') -# print(f'hidden_categories:{hidden_categories}') -# --- -cat_members = CatDepth("Association football players by nationality", sitecode="en", family="wikipedia", depth=0, ns="14") -# --- -print(f"{len(cat_members)=}") -# --- -red = page.page_links() -print(f"{len(red)=}") -# --- -# save = page.save(newtext='') -# api_new = NEW_API('en', family='wikipedia') -# login = api_new.Login_to_wiki() -# pages = api_new.Find_pages_exists_or_not(liste) -# pages = api_new.Get_Newpages() - - -# python3 core8/pwb.py newapi/wiki_page