From df0afd929b03e3a570bf13fe9503b48b886a748f Mon Sep 17 00:00:00 2001 From: Oussema Frikha <58266181+OussemaFr@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:19:12 +0400 Subject: [PATCH] feat: ship the facebook-scraper-api package (v0.1.0) facebook-scraper style functions (get_page_info, get_posts(account, pages=...), get_group_info, get_comments, search) and a command-line tool, built on socialapis-sdk. Listings follow end_cursor with a hard page ceiling; CSV export flattens nested fields. --- .github/workflows/release.yml | 38 +++++ .github/workflows/test.yml | 28 ++++ PACKAGE.md | 61 ++++++++ README.md | 23 +-- facebook_scraper_api/__init__.py | 224 +++++++++++++++++++++++++++++ facebook_scraper_api/__main__.py | 3 + facebook_scraper_api/_items.py | 69 +++++++++ facebook_scraper_api/_paging.py | 66 +++++++++ facebook_scraper_api/cli.py | 118 +++++++++++++++ facebook_scraper_api/py.typed | 0 pyproject.toml | 69 +++++++++ tests/test_facebook_scraper_api.py | 103 +++++++++++++ 12 files changed, 794 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 PACKAGE.md create mode 100644 facebook_scraper_api/__init__.py create mode 100644 facebook_scraper_api/__main__.py create mode 100644 facebook_scraper_api/_items.py create mode 100644 facebook_scraper_api/_paging.py create mode 100644 facebook_scraper_api/cli.py create mode 100644 facebook_scraper_api/py.typed create mode 100644 pyproject.toml create mode 100644 tests/test_facebook_scraper_api.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a98dff6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,38 @@ +name: Release + +# Tag `vX.Y.Z` publishes facebook-scraper-api to PyPI with trusted publishing (no token). +# +# Once: pypi.org → Your account → Publishing → Add a pending publisher: +# PyPI project facebook-scraper-api, owner SocialAPIsHub, +# repository facebook-scraper-python, workflow release.yml, environment pypi. + +on: + push: + tags: + - "v*.*.*" + +concurrency: + group: release + cancel-in-progress: false + +jobs: + pypi: + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/facebook-scraper-api/ + permissions: + id-token: write + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - run: pip install build + - run: python -m build + - name: Version matches tag + run: | + tag="${GITHUB_REF_NAME#v}" + version=$(python -c "import re;print(re.search(r'__version__ = \"(.+)\"', open('facebook_scraper_api/__init__.py').read()).group(1))") + test "$tag" = "$version" || { echo "::error::tag $tag != version $version"; exit 1; } + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b5881f0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - run: pip install -e ".[dev]" + - run: ruff check facebook_scraper_api tests + - run: ruff format --check facebook_scraper_api tests + - run: pytest -q tests diff --git a/PACKAGE.md b/PACKAGE.md new file mode 100644 index 0000000..2590278 --- /dev/null +++ b/PACKAGE.md @@ -0,0 +1,61 @@ +# facebook-scraper-api + +Public Facebook data from Python: Page details, Page and group posts, reels, videos, comments and search. Functions in the style of the old `facebook-scraper` (`get_posts(account, pages=...)`), but the data comes from a hosted API, so nothing breaks when Facebook changes its pages: **no login, no cookies, no browser, no proxies**. + +Powered by [SocialAPIs](https://socialapis.io). **[Get a free API key](https://socialapis.io/auth/signup)**: 200 calls/month, no card. + +```bash +pip install facebook-scraper-api +export SOCIALAPIS_TOKEN=your_key +``` + +```python +from facebook_scraper_api import get_page_info, get_posts, get_comments + +page = get_page_info("nasa") +print(page["title"], page["followers_count"]) + +for post in get_posts("nasa", pages=2, limit=9): # 2 calls, 9 posts each + print(post) + +for comment in get_comments("https://www.facebook.com/nasa/posts/...", pages=1): + print(comment) +``` + +Or call `set_api_token("...")` first instead of the environment variable. + +| Function | What it returns | +|---|---| +| `get_page_info(account)` | A Page's details: name, followers, likes, category, contact, links | +| `get_posts(account, pages=1, limit=None, after=None, before=None)` | A Page's posts, newest first | +| `get_group_posts(group, pages=1, ...)` / `get_posts(group=...)` | A public group's posts | +| `get_group_info(group)` | A public group's details | +| `get_reels(account, pages=1)` / `get_videos(account, pages=1)` | A Page's reels / videos | +| `get_post(post_url, extended=False)` | One post's details | +| `get_comments(post_url, pages=1)` | Comments on a post or reel | +| `search_posts(query, pages=1)` / `search_pages(query, pages=1)` | Keyword search | + +Listings are generators: they follow the API's `end_cursor` and stop after `pages` calls or when there are no more results. `account` is a Page URL or username; `group` is a group URL or id. For several keys or a custom connection, use `FacebookScraperAPI(api_token=...)`, which has the same methods. + +## Command line + +```bash +facebook-scraper-api page nasa +facebook-scraper-api posts nasa --pages 3 --limit 9 --out nasa.csv +facebook-scraper-api group-posts https://www.facebook.com/groups/123 --format jsonl +facebook-scraper-api comments https://www.facebook.com/nasa/posts/123 --pages 2 +``` + +CSV columns are the fields of the records (nested fields as `reactions.total`). + +## Costs + +Most calls cost 1 credit. Page and group posts cost round(limit ÷ 3) credits per call (limit 3-9, default 3). Failed calls (4xx / 5xx) are not charged. [Pricing](https://socialapis.io/pricing). + +## Links + +- [Endpoint reference](https://socialapis.io/api-sources/facebook-page-posts) · [Free Facebook follower count tool](https://socialapis.io/tools/facebook-follower-count) +- Source and migration guide from `facebook-scraper`: [github.com/SocialAPIsHub/facebook-scraper-python](https://github.com/SocialAPIsHub/facebook-scraper-python) +- Full SDK (Instagram, Marketplace, Ads Library, async): [`socialapis-sdk`](https://pypi.org/project/socialapis-sdk/) · Meta Ads Library only: [`facebook-ads-library-api`](https://pypi.org/project/facebook-ads-library-api/) + +Public data only. You are responsible for how you use it, including personal data rules (GDPR) where they apply. MIT license. diff --git a/README.md b/README.md index 1c5167d..259ee6f 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,35 @@ # facebook-scraper-python +[![facebook-scraper-api on PyPI](https://img.shields.io/pypi/v/facebook-scraper-api?label=facebook-scraper-api&logo=pypi&logoColor=white)](https://pypi.org/project/facebook-scraper-api/) [![PyPI](https://img.shields.io/pypi/v/socialapis-sdk?cacheSeconds=300&label=PyPI&logo=pypi&logoColor=white)](https://pypi.org/project/socialapis-sdk/) [![Python versions](https://img.shields.io/pypi/pyversions/socialapis-sdk)](https://pypi.org/project/socialapis-sdk/) [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) > Modern Python alternative to [`kevinzg/facebook-scraper`](https://github.com/kevinzg/facebook-scraper) — the 9.5k-star library that's been **abandoned since 2022**. Hosted backend means it doesn't break every time Meta changes their HTML. -This repo is a **migration landing page** + working examples. The actual SDK lives at [`SocialAPIsHub/socialapis-python`](https://github.com/SocialAPIsHub/socialapis-python) and ships as the `socialapis-sdk` package on PyPI. +This repo ships the **`facebook-scraper-api`** package: the same function names as `facebook-scraper` (`get_page_info`, `get_posts(account, pages=…)`, `get_group_info`, `get_comments`…), backed by a hosted API. It also has a command line tool. ```bash -pip install socialapis-sdk +pip install facebook-scraper-api +export SOCIALAPIS_TOKEN=your_key ``` ```python -from socialapis import FacebookScraper +from facebook_scraper_api import get_page_info, get_posts -fb = FacebookScraper(api_token="...") # alias of Facebook — keeps your import line greppable -page = fb.get_page_info("EngenSA") -print(page.title, page.followers_count, page.category) +page = get_page_info("EngenSA") +print(page["title"], page["followers_count"]) -for post in fb.get_page_posts("EngenSA").get("posts", []): - print(post["text"][:80]) +for post in get_posts("EngenSA", pages=2): # follows the cursor, 1 call per page + print(post) ``` +```bash +facebook-scraper-api posts EngenSA --pages 3 --limit 9 --out posts.csv +``` + +All functions, options and costs: [PACKAGE.md](PACKAGE.md). Need Instagram, Marketplace, the Ads Library or async too? The full SDK lives at [`SocialAPIsHub/socialapis-python`](https://github.com/SocialAPIsHub/socialapis-python) (`pip install socialapis-sdk`), and the rest of this page shows the migration with it. + **[Get a free API token →](https://socialapis.io/auth/signup)** — 200 calls/month, no credit card --- diff --git a/facebook_scraper_api/__init__.py b/facebook_scraper_api/__init__.py new file mode 100644 index 0000000..4c4f2eb --- /dev/null +++ b/facebook_scraper_api/__init__.py @@ -0,0 +1,224 @@ +"""Facebook scraper API for Python: public Pages, posts, groups, reels, comments and search. + +Works like the old ``facebook-scraper`` functions (``get_posts(account, pages=...)``) +but the data comes from a hosted API, so nothing breaks when Facebook changes +its pages: no login, no cookies, no browser, no proxies. Powered by +SocialAPIs (https://socialapis.io). + +Quick start:: + + from facebook_scraper_api import get_page_info, get_posts, set_api_token + + set_api_token("YOUR_API_TOKEN") # or set SOCIALAPIS_TOKEN + + print(get_page_info("nasa")["followers_count"]) + for post in get_posts("nasa", pages=2): + print(post) + +Free API key (200 calls/month, no card): https://socialapis.io/auth/signup +""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Iterator +from typing import Any + +from socialapis import Facebook + +from ._items import flatten, items_of, record_of +from ._paging import next_cursor, paginate + +__version__ = "0.1.0" + +__all__ = [ + "FacebookScraperAPI", + "__version__", + "flatten", + "get_comments", + "get_group_info", + "get_group_posts", + "get_page_info", + "get_post", + "get_posts", + "get_reels", + "get_videos", + "items_of", + "next_cursor", + "search_pages", + "search_posts", + "set_api_token", +] + + +class FacebookScraperAPI: + """All calls of this package on one client (one API key, one connection pool). + + Every API call costs credits (1 per call for most; Page / group posts + cost round(limit / 3)). ``pages`` caps how many calls a listing makes. + """ + + def __init__(self, api_token: str | None = None, *, client: Facebook | None = None) -> None: + if client is None: + token = api_token or os.environ.get("SOCIALAPIS_TOKEN") + if not token: + raise ValueError( + "Call set_api_token(...) or set SOCIALAPIS_TOKEN. Get a free key at " + "https://socialapis.io/auth/signup (200 calls/month, no card)." + ) + client = Facebook(api_token=token) + self._fb = client + + def __enter__(self) -> FacebookScraperAPI: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def close(self) -> None: + self._fb.close() + + def _items(self, call: Callable[..., dict[str, Any]], pages: int, **params: Any) -> Iterator[dict[str, Any]]: + for body in paginate(lambda cursor: call(end_cursor=cursor, **params), pages): + yield from items_of(body) + + # ---- single records ---------------------------------------------------- + + def get_page_info(self, account: str, **params: Any) -> dict[str, Any]: + """A Page's public details: name, followers, likes, category, contact, links.""" + return self._fb.get_page_info(account, **params).model_dump(exclude_none=True) + + def get_group_info(self, group: str, **params: Any) -> dict[str, Any]: + """A public group's details: name, members, privacy, description.""" + return self._fb.get_group_details(group, **params).model_dump(exclude_none=True) + + def get_post(self, post_url: str, *, extended: bool = False, **params: Any) -> dict[str, Any]: + """One post's details (``extended=True`` adds views, video URLs, music).""" + call = self._fb.get_post_details_extended if extended else self._fb.get_post_details + return record_of(call(post_url, **params)) + + # ---- listings ---------------------------------------------------------- + + def get_posts( + self, + account: str | None = None, + *, + group: str | None = None, + pages: int = 1, + limit: int | None = None, + after: str | None = None, + before: str | None = None, + **params: Any, + ) -> Iterator[dict[str, Any]]: + """Recent posts of a Page (``account``) or a public ``group``, newest first. + + ``limit`` is posts per call (3-9, default 3; cost round(limit / 3) + credits per call). ``after`` / ``before`` are ISO 8601 timestamps. + """ + if (account is None) == (group is None): + raise ValueError("give a Page (account) or a group, not both") + if limit is not None and not 3 <= limit <= 9: + raise ValueError("limit is posts per call, from 3 to 9") + target = account if account is not None else group + call = self._fb.get_page_posts if account is not None else self._fb.get_group_posts + yield from self._items( + lambda **kw: call(target, **kw), # type: ignore[arg-type] + pages, + limit=limit, + after_time=after, + before_time=before, + **params, + ) + + def get_reels(self, account: str, *, pages: int = 1, **params: Any) -> Iterator[dict[str, Any]]: + """A Page's reels.""" + yield from self._items(lambda **kw: self._fb.get_page_reels(account, **kw), pages, **params) + + def get_videos(self, account: str, *, pages: int = 1, **params: Any) -> Iterator[dict[str, Any]]: + """A Page's long-form videos.""" + yield from self._items(lambda **kw: self._fb.get_page_videos(account, **kw), pages, **params) + + def get_comments(self, post_url: str, *, pages: int = 1, **params: Any) -> Iterator[dict[str, Any]]: + """Top-level comments on a post or reel.""" + yield from self._items(lambda **kw: self._fb.get_post_comments(post_url, **kw), pages, **params) + + def search_posts(self, query: str, *, pages: int = 1, **params: Any) -> Iterator[dict[str, Any]]: + """Public posts matching a keyword.""" + yield from self._items(lambda **kw: self._fb.search_posts(query, **kw), pages, **params) + + def search_pages(self, query: str, *, pages: int = 1, **params: Any) -> Iterator[dict[str, Any]]: + """Pages matching a keyword.""" + yield from self._items(lambda **kw: self._fb.search_pages(query, **kw), pages, **params) + + +# --------------------------------------------------------------------------- +# Module-level functions, facebook-scraper style. They share one client, +# created on first use from set_api_token() or SOCIALAPIS_TOKEN. +# --------------------------------------------------------------------------- + +_default: FacebookScraperAPI | None = None + + +def set_api_token(api_token: str) -> None: + """Use this API key for the module-level functions.""" + global _default + if _default is not None: + _default.close() + _default = FacebookScraperAPI(api_token) + + +def _client() -> FacebookScraperAPI: + global _default + if _default is None: + _default = FacebookScraperAPI() + return _default + + +def get_page_info(account: str, **params: Any) -> dict[str, Any]: + """A Page's public details. See :meth:`FacebookScraperAPI.get_page_info`.""" + return _client().get_page_info(account, **params) + + +def get_group_info(group: str, **params: Any) -> dict[str, Any]: + """A public group's details. See :meth:`FacebookScraperAPI.get_group_info`.""" + return _client().get_group_info(group, **params) + + +def get_post(post_url: str, **params: Any) -> dict[str, Any]: + """One post's details. See :meth:`FacebookScraperAPI.get_post`.""" + return _client().get_post(post_url, **params) + + +def get_posts(account: str | None = None, **params: Any) -> Iterator[dict[str, Any]]: + """Posts of a Page or group. See :meth:`FacebookScraperAPI.get_posts`.""" + return _client().get_posts(account, **params) + + +def get_group_posts(group: str, **params: Any) -> Iterator[dict[str, Any]]: + """Posts of a public group.""" + return _client().get_posts(group=group, **params) + + +def get_reels(account: str, **params: Any) -> Iterator[dict[str, Any]]: + """A Page's reels.""" + return _client().get_reels(account, **params) + + +def get_videos(account: str, **params: Any) -> Iterator[dict[str, Any]]: + """A Page's long-form videos.""" + return _client().get_videos(account, **params) + + +def get_comments(post_url: str, **params: Any) -> Iterator[dict[str, Any]]: + """Comments on a post or reel.""" + return _client().get_comments(post_url, **params) + + +def search_posts(query: str, **params: Any) -> Iterator[dict[str, Any]]: + """Public posts matching a keyword.""" + return _client().search_posts(query, **params) + + +def search_pages(query: str, **params: Any) -> Iterator[dict[str, Any]]: + """Pages matching a keyword.""" + return _client().search_pages(query, **params) diff --git a/facebook_scraper_api/__main__.py b/facebook_scraper_api/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/facebook_scraper_api/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/facebook_scraper_api/_items.py b/facebook_scraper_api/_items.py new file mode 100644 index 0000000..37a7531 --- /dev/null +++ b/facebook_scraper_api/_items.py @@ -0,0 +1,69 @@ +"""Find the records in an answer and flatten them for CSV. + +Endpoints wrap their list under different keys (``posts``, ``items``, +``videos``, …), so instead of a per-endpoint path we take the longest list of +objects in the answer, outside our own ``meta`` / ``message`` envelope. The +same rule reads live answers in the SocialAPIs Google Sheets add-on. +""" + +from __future__ import annotations + +import json +from typing import Any + +from ._paging import ENVELOPE_KEYS + + +def _best_list(node: Any, depth: int) -> list[dict[str, Any]] | None: + if depth > 3: + return None + if isinstance(node, list): + if node and all(isinstance(x, dict) for x in node): + return node + return None + if not isinstance(node, dict): + return None + best: list[dict[str, Any]] | None = None + for key, value in node.items(): + if key in ENVELOPE_KEYS: + continue + found = _best_list(value, depth + 1) + if found is not None and (best is None or len(found) > len(best)): + best = found + return best + + +def items_of(body: Any) -> list[dict[str, Any]]: + """The records of one answer (posts, comments, videos…), or ``[]``.""" + return _best_list(body, 0) or [] + + +def record_of(body: Any) -> dict[str, Any]: + """A single-record answer without our envelope (and its ``"0"`` / ``data`` wrapper).""" + if not isinstance(body, dict): + return {} + for key in ("0", "data"): + inner = body.get(key) + if isinstance(inner, dict): + return inner + return {k: v for k, v in body.items() if k not in ENVELOPE_KEYS} + + +def _cell(value: Any) -> Any: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, list) and all(x is None or not isinstance(x, (dict, list)) for x in value): + return ", ".join(str(x) for x in value if x is not None) + return json.dumps(value, ensure_ascii=False) + + +def flatten(record: dict[str, Any], prefix: str = "", depth: int = 0) -> dict[str, Any]: + """``{"a": 1, "b": {"c": 2}}`` → ``{"a": 1, "b.c": 2}`` (two levels; deeper parts stay JSON).""" + out: dict[str, Any] = {} + for key, value in record.items(): + name = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, dict) and depth < 2: + out.update(flatten(value, name, depth + 1)) + else: + out[name] = _cell(value) + return out diff --git a/facebook_scraper_api/_paging.py b/facebook_scraper_api/_paging.py new file mode 100644 index 0000000..befa6a6 --- /dev/null +++ b/facebook_scraper_api/_paging.py @@ -0,0 +1,66 @@ +"""Cursor handling shared by every paginated call. + +The API returns its next-page cursor as ``end_cursor`` (usually inside a +``page_info`` block next to ``has_next_page``). Where exactly it sits differs +by endpoint, so we look for it instead of hard-coding a path, and stop as soon +as there is no cursor, ``has_next_page`` is false, or a cursor repeats. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from typing import Any + +# Our own envelope around every answer: never data, never a cursor. +ENVELOPE_KEYS = ("meta", "message") + +_STOP = object() + + +def _find_cursor(node: Any, depth: int) -> Any: + if depth > 4 or not isinstance(node, dict): + return None + info = node.get("page_info") + if isinstance(info, dict): + if info.get("has_next_page") is False: + return _STOP + cursor = info.get("end_cursor") + if isinstance(cursor, str) and cursor: + return cursor + cursor = node.get("end_cursor") + if isinstance(cursor, str) and cursor: + return cursor + for key, value in node.items(): + if key in ENVELOPE_KEYS: + continue + found = _find_cursor(value, depth + 1) + if found is not None: + return found + return None + + +def next_cursor(body: Any) -> str | None: + """The cursor for the next page, or None when there is no next page.""" + found = _find_cursor(body, 0) + return found if isinstance(found, str) else None + + +def paginate( + fetch: Callable[[str | None], dict[str, Any]], + pages: int, +) -> Iterator[dict[str, Any]]: + """Call ``fetch(cursor)`` up to ``pages`` times, yielding each answer. + + Every call is a billed API call, so ``pages`` is a hard ceiling. + """ + if pages < 1: + raise ValueError("pages must be 1 or more") + cursor: str | None = None + seen: set[str] = set() + for _ in range(pages): + body = fetch(cursor) + yield body + cursor = next_cursor(body) + if cursor is None or cursor in seen: + return + seen.add(cursor) diff --git a/facebook_scraper_api/cli.py b/facebook_scraper_api/cli.py new file mode 100644 index 0000000..d35ac61 --- /dev/null +++ b/facebook_scraper_api/cli.py @@ -0,0 +1,118 @@ +"""Command line: Facebook Pages, posts, groups, reels and comments to CSV or JSON lines. + + facebook-scraper-api page nasa + facebook-scraper-api posts nasa --pages 3 --limit 9 --out nasa.csv + facebook-scraper-api group-posts https://www.facebook.com/groups/123 --format jsonl + facebook-scraper-api comments https://www.facebook.com/nasa/posts/123 --pages 2 + +The API key comes from --token or the SOCIALAPIS_TOKEN environment variable. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from collections.abc import Iterable +from typing import Any, TextIO + +from socialapis import SocialAPIsError + +from . import FacebookScraperAPI, __version__ +from ._items import flatten + +LISTINGS = { + "posts": ("Page posts", "Page URL or username"), + "group-posts": ("public group posts", "group URL or id"), + "reels": ("Page reels", "Page URL or username"), + "videos": ("Page videos", "Page URL or username"), + "comments": ("comments on a post or reel", "post URL"), + "search-posts": ("public posts matching a keyword", "keyword"), + "search-pages": ("Pages matching a keyword", "keyword"), +} + + +def write_rows(records: Iterable[dict[str, Any]], fmt: str, out: TextIO) -> int: + """CSV (headers = every field seen, in order) or one JSON object per line.""" + records = list(records) + if fmt == "jsonl": + for record in records: + out.write(json.dumps(record, ensure_ascii=False) + "\n") + return len(records) + flat = [flatten(r) for r in records] + headers: list[str] = [] + for row in flat: + headers.extend(k for k in row if k not in headers) + writer = csv.DictWriter(out, fieldnames=headers or ["(no results)"]) + writer.writeheader() + writer.writerows(flat) + return len(flat) + + +def _listing(api: FacebookScraperAPI, args: argparse.Namespace) -> Iterable[dict[str, Any]]: + common: dict[str, Any] = {"pages": args.pages} + if args.command in ("posts", "group-posts"): + common.update(limit=args.limit, after=args.after, before=args.before) + if args.command == "posts": + return api.get_posts(args.target, **common) + return api.get_posts(group=args.target, **common) + calls = { + "reels": api.get_reels, + "videos": api.get_videos, + "comments": api.get_comments, + "search-posts": api.search_posts, + "search-pages": api.search_pages, + } + return calls[args.command](args.target, **common) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="facebook-scraper-api", + description="Public Facebook data (Pages, posts, groups, reels, comments) without a login or browser.", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + parser.add_argument("--token", help="SocialAPIs API key (default: $SOCIALAPIS_TOKEN)") + sub = parser.add_subparsers(dest="command", required=True) + + for name in ("page", "group"): + p = sub.add_parser(name, help=f"a {name}'s details (JSON)") + p.add_argument("target", help="URL, username or id") + + for name, (help_text, target_help) in LISTINGS.items(): + p = sub.add_parser(name, help=help_text) + p.add_argument("target", help=target_help) + p.add_argument("--pages", type=int, default=1, help="API calls to make (default 1)") + if name in ("posts", "group-posts"): + p.add_argument("--limit", type=int, help="posts per call, 3-9 (cost round(limit / 3) credits per call)") + p.add_argument("--after", help="only posts after this time (ISO 8601)") + p.add_argument("--before", help="only posts before this time (ISO 8601)") + p.add_argument("--format", choices=["csv", "jsonl"], default="csv") + p.add_argument("--out", help="file to write (default: standard output)") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + with FacebookScraperAPI(api_token=args.token) as api: + if args.command in ("page", "group"): + info = api.get_page_info(args.target) if args.command == "page" else api.get_group_info(args.target) + print(json.dumps(info, indent=2, ensure_ascii=False)) + return 0 + records = _listing(api, args) + if args.out: + with open(args.out, "w", newline="", encoding="utf-8") as fh: + count = write_rows(records, args.format, fh) + print(f"{count} rows written to {args.out}", file=sys.stderr) + else: + write_rows(records, args.format, sys.stdout) + except (SocialAPIsError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/facebook_scraper_api/py.typed b/facebook_scraper_api/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..93c451a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["hatchling>=1.21"] +build-backend = "hatchling.build" + +[project] +name = "facebook-scraper-api" +dynamic = ["version"] +description = "Facebook scraper API for Python: public Pages, posts, groups, reels, comments and search, with no login, cookies or browser. facebook-scraper style functions." +readme = "PACKAGE.md" +requires-python = ">=3.10" +authors = [{ name = "SocialAPIs", email = "pypi@socialapis.io" }] +license = { text = "MIT" } +keywords = [ + "facebook-scraper", + "facebook-scraper-api", + "facebook-api", + "facebook-posts", + "facebook-pages", + "facebook-groups", + "facebook-comments", + "scraping", + "social-media", + "socialapis", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Internet :: WWW/HTTP", + "Typing :: Typed", +] +dependencies = ["socialapis-sdk>=0.1.3"] + +[project.scripts] +facebook-scraper-api = "facebook_scraper_api.cli:main" + +[project.urls] +Homepage = "https://socialapis.io" +Documentation = "https://socialapis.io/api-sources/facebook-page-posts" +Pricing = "https://socialapis.io/pricing" +"Free follower count tool" = "https://socialapis.io/tools/facebook-follower-count" +Repository = "https://github.com/SocialAPIsHub/facebook-scraper-python" +Issues = "https://github.com/SocialAPIsHub/facebook-scraper-python/issues" + +[project.optional-dependencies] +dev = ["pytest>=8", "ruff>=0.6"] + +[tool.hatch.version] +path = "facebook_scraper_api/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["facebook_scraper_api"] + +[tool.hatch.build.targets.sdist] +include = ["/facebook_scraper_api", "/tests", "/PACKAGE.md", "/LICENSE"] + +[tool.ruff] +target-version = "py310" +line-length = 120 +extend-exclude = ["examples/"] + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B", "SIM"] diff --git a/tests/test_facebook_scraper_api.py b/tests/test_facebook_scraper_api.py new file mode 100644 index 0000000..1fb4fe6 --- /dev/null +++ b/tests/test_facebook_scraper_api.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import csv +import io +from typing import Any + +import pytest + +from facebook_scraper_api import FacebookScraperAPI, flatten, items_of +from facebook_scraper_api.cli import write_rows + +POST_A = {"post_id": "1", "message": "Hello", "reactions": {"total": 5, "like": 4}} +POST_B = {"post_id": "2", "message": "World", "attachments": [{"type": "photo"}]} + + +def answer(posts: list[dict[str, Any]], cursor: str | None) -> dict[str, Any]: + return { + "message": "ok", + "meta": {"statusCode": 200, "creditsCharged": 1}, + "data": {"posts": posts, "page_info": {"end_cursor": cursor, "has_next_page": cursor is not None}}, + } + + +class FakeFacebook: + def __init__(self, answers: list[dict[str, Any]]) -> None: + self.answers = answers + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + + def _answer(self, name: str, target: str, kwargs: dict[str, Any]) -> dict[str, Any]: + self.calls.append((name, target, kwargs)) + return self.answers[len(self.calls) - 1] + + def get_page_posts(self, page: str, **kwargs: Any) -> dict[str, Any]: + return self._answer("page_posts", page, kwargs) + + def get_group_posts(self, group: str, **kwargs: Any) -> dict[str, Any]: + return self._answer("group_posts", group, kwargs) + + def get_post_comments(self, post: str, **kwargs: Any) -> dict[str, Any]: + return self._answer("comments", post, kwargs) + + def close(self) -> None: + pass + + +def test_posts_follow_the_cursor() -> None: + fb = FakeFacebook([answer([POST_A], "c1"), answer([POST_B], None)]) + api = FacebookScraperAPI(client=fb) # type: ignore[arg-type] + posts = list(api.get_posts("nasa", pages=5, limit=9)) + assert [p["post_id"] for p in posts] == ["1", "2"] + assert fb.calls[0] == ( + "page_posts", + "nasa", + {"end_cursor": None, "limit": 9, "after_time": None, "before_time": None}, + ) + assert fb.calls[1][2]["end_cursor"] == "c1" + + +def test_pages_is_a_ceiling() -> None: + fb = FakeFacebook([answer([POST_A], "c1"), answer([POST_B], "c2")]) + list(FacebookScraperAPI(client=fb).get_posts("nasa", pages=1)) # type: ignore[arg-type] + assert len(fb.calls) == 1 + + +def test_group_posts() -> None: + fb = FakeFacebook([answer([POST_A], None)]) + list(FacebookScraperAPI(client=fb).get_posts(group="123")) # type: ignore[arg-type] + assert fb.calls[0][:2] == ("group_posts", "123") + + +def test_post_arguments_are_checked() -> None: + api = FacebookScraperAPI(client=FakeFacebook([])) # type: ignore[arg-type] + with pytest.raises(ValueError, match="not both"): + list(api.get_posts("nasa", group="123")) + with pytest.raises(ValueError, match="not both"): + list(api.get_posts()) + with pytest.raises(ValueError, match="3 to 9"): + list(api.get_posts("nasa", limit=20)) + + +def test_items_of_takes_the_longest_list_outside_the_envelope() -> None: + body = { + "meta": {"warnings": [{"a": 1}, {"b": 2}, {"c": 3}]}, + "data": {"items": [POST_A, POST_B], "page_info": {}}, + } + assert items_of(body) == [POST_A, POST_B] + assert items_of({"data": [POST_A]}) == [POST_A] + assert items_of({"data": {"posts": []}}) == [] + + +def test_flatten_and_csv() -> None: + assert flatten(POST_A) == {"post_id": "1", "message": "Hello", "reactions.total": 5, "reactions.like": 4} + out = io.StringIO() + assert write_rows([POST_A, POST_B], "csv", out) == 2 + rows = list(csv.DictReader(io.StringIO(out.getvalue()))) + assert rows[1]["attachments"] == '[{"type": "photo"}]' + assert rows[0]["reactions.total"] == "5" + + +def test_missing_token(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("SOCIALAPIS_TOKEN", raising=False) + with pytest.raises(ValueError, match="SOCIALAPIS_TOKEN"): + FacebookScraperAPI()