diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..541e91d --- /dev/null +++ b/.cursorrules @@ -0,0 +1,73 @@ +# SerpApi Python Client - AI Assistant Configuration + +## Project Overview +This is a Python client library for SerpApi.com - Search Engine Results API. It's a migration from Ruby to Python with async/await support. + +## Key Technologies +- Python 3.11+ +- Async/await with aiohttp +- Type hints with mypy +- UV package manager +- Black and isort for code formatting +- Pytest for testing + +## Project Structure +``` +serpapi/ +├── __init__.py # Main package exports +├── client.py # Async HTTP client +├── error.py # Custom exceptions +└── version.py # Version information + +tests/ +├── test_client.py # Client functionality tests +└── test_error.py # Error handling tests + +examples/ +├── basic_usage.py # Basic usage examples +├── async_batch_search.py # Batch processing examples +└── multiple_engines.py # Multi-engine examples +``` + +## Development Workflow +- Use `uv sync --dev` to update dependencies +- Use `uv run pytest` to run tests +- Use `uv run mypy serpapi/` to run type checking +- Use `uv run black .` to format code +- Use `uv run isort .` to sort imports +- Use `uv run python script.py` to run Python scripts + + +## Code Style Guidelines +- Follow PEP 8 for Python code +- Use type hints for all functions +- Prefer async/await over blocking calls +- Use f-strings for string formatting +- Keep functions focused and testable + +## Important Files +- `serpapi/client.py` - Main client implementation +- `README.md` - User documentation +- `pyproject.toml` - Project configuration and tool settings +- `MIGRATION_SUMMARY.md` - Ruby to Python migration details + +## API Key Configuration +Set environment variable: `export SERPAPI_KEY="your_api_key"` + +## Testing Strategy +- Unit tests for individual components +- Integration tests for API calls +- Type checking with mypy +- Import testing for basic functionality + +## Common Patterns +- Use context managers for client lifecycle +- Handle SerpApiError exceptions +- Use persistent connections for better performance +- Batch multiple requests with asyncio.gather() + +## Migration Notes +- Converted from Ruby gem to Python package +- Maintained API compatibility where possible +- Added async support for better performance +- Used modern Python packaging with pyproject.toml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index d3bcd4e..0000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,46 +0,0 @@ - - -## Summary - - - -## Type - -- [ ] Bug fix -- [ ] Feature -- [ ] Refactor / performance -- [ ] Documentation / examples -- [ ] Tests -- [ ] Maintenance / packaging / CI - -## Related issue - - - -## Affected SDK functionality - -- `serpapi` package version (if known): -- Python version and operating system: -- SDK methods or helpers affected: -- SerpApi engine or endpoint involved (if applicable): -- Other affected Python versions or environments (confirmed or suspected): - -## Testing - - - -- Human testing: -- AI-executed testing: -- CI results, failures, or tests skipped (with reasons): - -## AI disclosure - - - -- Model(s): -- Harness/app(s) (e.g., Codex, Claude Code, Cursor): -- Used for (e.g., coding, tests, review, documentation): - -## Evidence - - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 8d19c8e..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: serpapi-python - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - - - name: Install dependencies - run: pip install -e .[test] - - - name: Run tests - run: pytest tests --ignore-glob='tests/test_docs_*.py' -k "not example" - env: - API_KEY: ${{ secrets.API_KEY }} - - - name: Run example tests - if: matrix.python-version == '3.15' - run: pytest tests/example_search_*_test.py - env: - API_KEY: ${{ secrets.API_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f147be5..0000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: Documentation - -on: - push: - tags: ['v**'] - workflow_dispatch: - -permissions: - contents: read - -jobs: - live-examples: - name: Live documentation examples - if: >- - github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - uses: astral-sh/setup-uv@v7 - - - name: Install test dependencies - run: | - uv venv - uv pip install -e '.[test]' - - - name: Run live documentation examples - run: >- - .venv/bin/python -m pytest tests/test_docs_examples.py - --require-docs-key -q --junitxml=docs-example-results.xml - env: - API_KEY: ${{ secrets.API_KEY }} - - - uses: actions/upload-artifact@v7 - if: always() - with: - name: docs-example-results - path: docs-example-results.xml - if-no-files-found: warn - - build: - name: Build Sphinx documentation - needs: [live-examples] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - uses: astral-sh/setup-uv@v7 - - - name: Install documentation dependencies - run: | - uv venv - uv pip install -e '.[docs,test]' - - - name: Test documentation publishing - run: .venv/bin/python -m pytest tests/test_docs_publishing.py -q - - - name: Check documentation code and test runner without API calls - run: >- - .venv/bin/python -m pytest tests/test_docs_examples.py - tests/test_docs_example_runner.py -k 'not live' -q - - - name: Build HTML documentation - run: .venv/bin/sphinx-build -M html docs docs/_build -W --keep-going - - - name: Build EPUB documentation - run: .venv/bin/sphinx-build -M epub docs docs/_build -W --keep-going - - - uses: actions/upload-artifact@v7 - with: - name: documentation - path: docs/_build/ - if-no-files-found: error - - publish: - name: Publish Read the Docs - needs: [live-examples, build] - if: >- - needs.live-examples.result == 'success' && - needs.build.result == 'success' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && - (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) - runs-on: ubuntu-latest - timeout-minutes: 60 - environment: - name: docs - url: https://serpapi-python.readthedocs.io/ - concurrency: - group: readthedocs-publish - cancel-in-progress: false - queue: max - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ github.sha }} - persist-credentials: false - - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Sync versions, publish, and wait for RTD - run: python -m scripts.publish_docs - env: - RTD_API_TOKEN: ${{ secrets.RTD_API_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 7e37094..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: Release - -on: - push: - tags: ['v**'] - workflow_dispatch: - inputs: - allow_example_test_failures: - description: 'Publish even if the live example tests fail (use only when a SerpApi engine is down)' - type: boolean - default: false - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -jobs: - test: - name: Test / Python ${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] - steps: - - uses: actions/checkout@v6 - - - name: Ensure ref is a tag - if: github.ref_type != 'tag' - run: | - echo "Release must run against a tag, got '${{ github.ref }}'" >&2 - exit 1 - - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: pip install -e .[test] - - - name: Run tests - run: pytest --ignore=tests/test_docs_publishing.py -k "not example" - env: - API_KEY: ${{ secrets.API_KEY }} - - - name: Run engine and documentation examples - if: matrix.python-version == '3.15' - continue-on-error: ${{ inputs.allow_example_test_failures == true }} - run: pytest --ignore=tests/test_docs_publishing.py -k "example" --require-docs-key - env: - API_KEY: ${{ secrets.API_KEY }} - - build: - name: Build distribution - needs: [test] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Build sdist and wheel - run: | - pip install build - python -m build - - - uses: actions/upload-artifact@v7 - with: - name: dist - path: dist/ - if-no-files-found: error - - publish: - name: Publish release - needs: [build] - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/serpapi - permissions: - contents: write - id-token: write - steps: - - uses: actions/download-artifact@v8 - with: - name: dist - path: dist/ - - - name: Create GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - run: | - gh release create ${{ github.ref_name }} dist/* --generate-notes \ - || gh release upload ${{ github.ref_name }} dist/* --clobber - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - smoke-test: - name: Smoke test published package - needs: [publish] - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Wait for PyPI availability - run: | - VERSION=${GITHUB_REF_NAME#v} - echo "Waiting for serpapi==$VERSION on PyPI..." - for i in $(seq 1 12); do - if pip index versions serpapi 2>/dev/null | grep -q "$VERSION"; then - echo "Available!" - exit 0 - fi - echo "Attempt $i/12 — sleeping 10s..." - sleep 10 - done - echo "Timed out waiting for PyPI propagation" && exit 1 - - - name: Install from PyPI - run: | - VERSION=${GITHUB_REF_NAME#v} - pip install "serpapi==$VERSION" - - - name: Verify import and version - env: - API_KEY: ${{ secrets.API_KEY }} - EXPECTED_VERSION: ${{ github.ref_name }} - shell: python3 {0} - run: | - import os - import serpapi - expected = os.environ["EXPECTED_VERSION"].lstrip("v") - assert serpapi.__version__ == expected, f"Version mismatch: {serpapi.__version__} != {expected}" - print(f"OK: serpapi=={serpapi.__version__} installed successfully") - client = serpapi.Client(api_key=os.environ["API_KEY"]) - results = client.search({"engine": "google", "q": "coffee"}) - assert results.get("organic_results"), "No organic results returned" - print(f"OK: live search returned {len(results['organic_results'])} organic results") diff --git a/.gitignore b/.gitignore index f08db2c..39a54eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,8 @@ -.vscode/** +## PROJECT::GENERAL +__pycache__ -script/ -.coverage -docs/_build/ -dist/ -build/ -.pytest_cache/ serpapi.egg-info/ - - -__pycache__ -*.pyc +.vscode/ +.coverage .DS_Store -.envrc - -.vscode -t.py -.venv/ +uv.lock diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..2c07333 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index 31b038d..0000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,27 +0,0 @@ -version: 2 - -build: - os: ubuntu-24.04 - tools: - python: "3.13" - jobs: - pre_build: - - python -m scripts.check_docs_revision - post_build: - - python -m scripts.check_docs_revision - -sphinx: - configuration: docs/conf.py - fail_on_warning: true - -formats: - - pdf - - epub - -python: - install: - - method: uv - command: pip - path: . - extras: - - docs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 74afa96..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,183 +0,0 @@ -# Contributing - -Bug reports and pull requests are welcome on GitHub. Run commands from the repository root. - -## Development setup - -The SDK CI tests Python 3.8 through 3.15. Create a virtual environment and install the package with its test and documentation dependencies: - -```sh -uv venv --python 3.13 -uv pip install -e '.[test,docs]' -``` - -The editable install uses the Python files in your checkout, so source changes take effect without reinstalling the package. - -Activate the environment on macOS or Linux: - -```sh -source .venv/bin/activate -``` - -On Windows PowerShell: - -```powershell -.\.venv\Scripts\Activate.ps1 -``` - -The commands below use `python` from that environment. - -## Testing - -### Checks that need no API key - -These tests cover response parsing, image uploads and extraction with simulated responses, timeouts, exceptions, documentation syntax, and the example runner: - -```sh -python -m pytest tests/test_output_formats.py tests/test_image_upload.py tests/test_extract.py tests/test_timeout.py tests/test_exceptions.py tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q -``` - -The file list limits this command to tests that make no external requests. Applying `-k 'not live'` to the entire suite is not enough to run offline: some SDK integration tests do not have `live` in their names. - -### SDK integration tests - -Set `API_KEY` in your shell or editor's run configuration before running integration tests. Documentation examples also accept `SERPAPI_KEY`, but the shared SDK fixtures require `API_KEY`. If you already have `SERPAPI_KEY` set, copy it to `API_KEY` in the same shell: - -```sh -export API_KEY="$SERPAPI_KEY" -``` - -In PowerShell, use `$env:API_KEY = $env:SERPAPI_KEY`. Keep real keys out of source files and commits. - -To match the test selection in the SDK CI workflow: - -```sh -python -m pytest tests --ignore-glob='tests/test_docs_*.py' -k 'not example' -q -``` - -This includes live account, location, search, and pagination checks. To run every discovered test, including the standalone engine examples and documentation examples: - -```sh -python -m pytest -q -``` - -Both commands make real SerpApi requests. The full suite can use more searches than a test run limited to the files you changed. PR CI also runs the standalone engine examples on Python 3.15. Documentation tests run in the package and documentation release workflows on release tags, and in manual documentation runs on `master`. - -### Testing documentation examples - -The tests discover Python code blocks in the README and all Markdown pages under `docs/`. They run each page as a separate script against SerpApi, including the multiprocessing example's worker processes. The Lens upload examples use the repository's PNG icon as `image.png`. Each page has a five-minute limit, and HTTP errors or JSON responses containing an `error` fail the check even if the example catches the exception. - -With `SERPAPI_KEY` or `API_KEY` set in your environment, run: - -```sh -python -m pytest tests/test_docs_examples.py --require-docs-key -q -``` - -The `--require-docs-key` option fails if neither key is set. Without that option, local runs skip live tests when no key is available. To check syntax and the test runner without making API calls: - -```sh -python -m pytest tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q -``` - -Code blocks marked with `docs-test: skip` are checked for syntax but not executed. These cover the old `google-search-results` package and examples that require private proxy or certificate settings, or disable TLS verification. Each marker includes its reason. - -### Testing documentation publishing - -Run the publishing tests locally with: - -```sh -python -m pytest tests/test_docs_publishing.py -q -``` - -These tests use simulated RTD responses to check outgoing HTTP requests, CI event and commit checks, version selection, polling, and cleanup after failures. They need no API keys and do not publish documentation. In CI, they run only in the Documentation workflow on Python 3.13, before the documentation build and publication. The SDK and PyPI release workflows exclude this file. RTD checks the actual checkout with `scripts.check_docs_revision` before and after building, without rerunning the publishing tests. - -### Running a single test or example - -Run one test file while working on that part of the package: - -```sh -python -m pytest tests/test_output_formats.py -q -``` - -To list documentation test IDs without running their examples: - -```sh -python -m pytest tests/test_docs_examples.py --collect-only -q -``` - -For example, run only the Google Lens upload and search page with: - -```sh -python -m pytest 'tests/test_docs_examples.py::test_documentation_examples_live[docs/examples/google-lens-image-upload.md]' --require-docs-key -q -``` - -Use `-x -vv` in place of `-q` to stop at the first failure and show more detail. A syntax failure points to a page and block number. A live failure can come from the example code, an invalid key, exhausted quota, or an upstream API error; inspect the reported engine and error before changing the example. - -When adding a documentation example, use a fenced block labelled `python`. Blocks on the same page execute in order and share variables. Avoid hard-coded dates that expire. Use a `docs-test: skip` marker with a reason only when a block cannot run in the test environment, such as one requiring a user's proxy or certificate. - -## Building the documentation locally - -The documentation uses Sphinx with MyST for Markdown pages and the Read the Docs theme. The development setup above includes its dependencies. Build and serve the HTML site with: - -```sh -python -m sphinx -M html docs docs/_build -W --keep-going -python -m http.server 8000 --bind 127.0.0.1 --directory docs/_build/html -``` - -Open [the local documentation](http://127.0.0.1:8000). After editing a page, rerun the Sphinx build command and refresh your browser. Check the affected page, its code blocks, and the sidebar links. Press Ctrl+C to stop the server. - -Sphinx discovers public APIs with autodoc and reads their docstrings without running the search examples. Sidebar order comes from the toctrees in `docs/index.md`, so documentation filenames do not need numeric prefixes. The `-W` option makes warnings fail the build. - -CI also builds EPUB. Check it locally with: - -```sh -python -m sphinx -M epub docs docs/_build -W --keep-going -``` - -## Building the package - -Build the source distribution and wheel with: - -```sh -uv build -``` - -The files are written to `dist/`. Documentation sources, this guide, and the logo assets are included in the source distribution (`.tar.gz`) so contributors can build the docs from a source release. The wheel contains only the `serpapi` library and its package metadata. Generated docs and documentation dependencies are not part of a normal installation. - -## Documentation publishing - -The [documentation workflow](.github/workflows/docs.yml) runs the live examples before building HTML and EPUB when a `v*` release tag is pushed, or when `master` or a release tag is selected for a manual run. It does not run on branch pushes or pull requests. - -After the live examples and documentation build pass, the same workflow publishes to Read the Docs for the selected ref. A manual run on `master` updates `latest` without triggering the package release workflow or changing the package version. The publishing job uses the `docs` GitHub environment and its `RTD_API_TOKEN` secret. The SerpApi key stays in GitHub as the existing `API_KEY` repository or organization secret. - -The workflow syncs RTD versions, activates the requested version if needed, and waits for the build to finish. `latest` tracks `master`. A release tag has its own version and also rebuilds both `stable` and `latest` when RTD identifies it as the highest stable release. Both builds must use the release commit tested by CI. If `master` has advanced beyond that commit, the `latest` build fails the revision check; run the Documentation workflow on `master` to test and publish its current docs. Older releases and prereleases do not update either alias. RTD still builds the site from the repository using [.readthedocs.yaml](.readthedocs.yaml); it does not receive the HTML artifact from GitHub. Documentation publishing runs independently of the PyPI release workflow. - -Before requesting a build, the workflow creates a temporary RTD environment variable named `DOCS_CI_REVISION`. It contains the tested commit, the permitted versions, and an expiration time. RTD checks this record against its checkout before and after the Sphinx build. A missing, expired, or different revision stops publication. RTD needs no SerpApi key. The workflow removes the temporary record after publishing, including when a build fails. Publishing jobs run one at a time so they cannot overwrite each other's revision record. - -### Maintainer setup - -Maintainers can configure the existing RTD project and GitHub environment with these steps: - -1. In RTD **Settings**, set **Connected repository** to **No connected repository** and keep **Repository URL** set to `https://github.com/serpapi/serpapi-python.git`. Set **Default branch** to `master` and the configuration file path to `.readthedocs.yaml`. The public repository URL lets RTD clone the source without receiving GitHub push events through the GitHub App. -2. Under RTD **Integrations**, remove incoming GitHub webhook integrations for this project. If an older RTD webhook is also listed in the GitHub repository's **Settings > Webhooks**, disable or remove that webhook. Do not remove integrations for other projects. -3. Under RTD **Automation Rules**, remove rules that activate new versions or change the default version. The workflow handles release activation. Under **Settings > Pull request builds**, turn off **Build pull requests for this project**. GitHub Actions runs the existing SDK and engine example tests on PRs. -4. Under RTD **Environment Variables**, remove `API_KEY` or `SERPAPI_KEY` if you added either for docs tests. Do not add the RTD API token here. The workflow manages `DOCS_CI_REVISION` automatically. -5. Keep `latest` active in **Versions** and use it as the default documentation version during this migration. Existing release tags contain their original docs and build configuration. After the first release containing these changes builds successfully, you can choose `stable` as the default version. -6. Create an RTD API token in your [RTD profile settings](https://app.readthedocs.org/accounts/tokens/), using an account that maintains the `serpapi-python` project. In GitHub, open the repository's **Settings > Environments**, create an environment named `docs`, and add an environment secret named `RTD_API_TOKEN` with that value. Under **Deployment branches and tags**, select **Selected branches and tags** and add a Branch rule for `master` and a Tag rule for `v*`. Leave required reviewers and wait timers disabled if publishing should run without a manual approval. -7. Merge the changes to `master`, then push the version tag for the release. In GitHub **Actions > Documentation**, follow **Live documentation examples**, **Build Sphinx documentation**, and **Publish Read the Docs**. The publishing log links to the RTD build. To retry publishing, run the Documentation workflow on the intended release tag, or select `master` to publish updated `latest` docs without a new package release. A manual run on any other branch skips the documentation jobs. - -Use GitHub Actions to request builds after this setup. A manual RTD build has no CI revision record and will fail the revision check. If a branch or tag moves between testing and the RTD checkout, rerun the workflow for its current commit. Pushing a `v*` tag also starts the PyPI release workflow, so use an actual package release to test release documentation. - -See the [RTD build API](https://docs.readthedocs.com/platform/stable/api/v3.html#build-triggering), [Git integration settings](https://docs.readthedocs.com/platform/stable/reference/git-integration.html), and [GitHub environment settings](https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/manage-environments) for the platform setup details. - -## Publishing a new release - -1. Update the version in `serpapi/__version__.py`. -2. Push a tag for that version. The release pipeline runs automatically: - ```sh - git tag v1.2.3 - git push origin v1.2.3 - ``` - This triggers the [release workflow](.github/workflows/release.yml), which tests, builds, and publishes to PyPI, then smoke-tests the published package. - -> **Required secret:** `API_KEY` (used by the live documentation examples and the published-package smoke test). diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..5df41f1 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,162 @@ +# SerpApi Python Library + +Official Python client for SerpApi.com - Search Engine Results API. + +## Features + +- Async/await support for non-blocking HTTP requests +- Persistent connections for 2x faster response times +- Multiple search engines (Google, Bing, Yahoo, Baidu, Yandex, etc.) +- Comprehensive API coverage (Search, Location, Account, Search Archive) +- Type hints and extensive documentation + +## Installation + + +```bash +# Using pip +pip install serpapi + +# Using uv (recommended) +uv add serpapi +``` + +## Quick Start + +```python +import asyncio +from serpapi import Client + +async def main(): + client = Client(api_key="your_api_key", engine="google") + results = await client.search({"q": "coffee"}) + + for result in results.get("organic_results", []): + print(f"Title: {result.get('title')}") + print(f"Link: {result.get('link')}") + + await client.close() + +asyncio.run(main()) +``` + +## API Key + +Get your API key from [serpapi.com/signup](https://serpapi.com/users/sign_up?plan=free). + +Set environment variable: +```bash +export SERPAPI_KEY="your_secret_key" +``` + +## Usage Examples + +### Basic Search +```python +results = await client.search({"q": "coffee"}) +``` + +### HTML Search +```python +html_content = await client.html({"q": "coffee"}) +``` + +### Location API +```python +locations = await client.location({"q": "Austin", "limit": 3}) +``` + +### Search Archive +```python +archived = await client.search_archive(search_id) +``` + +### Account Info +```python +account = await client.account() +``` + +## Async Batch Processing + +```python +import asyncio + +async def search_company(client, company): + results = await client.search({"q": company}) + return {"company": company, "count": len(results.get("organic_results", []))} + +async def main(): + client = Client(api_key="your_api_key", persistent=True) + companies = ["meta", "amazon", "apple", "netflix", "google"] + + tasks = [search_company(client, company) for company in companies] + results = await asyncio.gather(*tasks) + + for result in results: + print(f"{result['company']}: {result['count']} results") + + await client.close() + +asyncio.run(main()) +``` + +## Context Manager + +```python +async with Client(api_key="your_api_key") as client: + results = await client.search({"q": "coffee"}) + # Client automatically closed +``` + +## Error Handling + +```python +from serpapi import SerpApiError + +try: + results = await client.search({"q": "coffee"}) +except SerpApiError as e: + print(f"SerpApi error: {e}") +``` + +## Developer guide + +The UV Package Manager must be installed. See [uv installation instructions](https://docs.astral.sh/uv/getting-started/installation). + +The following commands are available: + +```bash +# Install with development dependencies +uv sync --dev + +# Run tests +uv run pytest + +# Run test with coverage +uv run pytest --cov=serpapi tests/ +``` + +### UV Benefits + +- **Fast**: 10-100x faster than pip +- **Reliable**: Lock file ensures reproducible builds +- **Simple**: Single command for most operations +- **Modern**: Built for Python 3.11+ with async support + +### Project Structure with UV + +``` +serpapi-python/ +├── .python-version # Python version (3.11) +├── uv.lock # Dependency lock file +├── .venv/ # Virtual environment (auto-created) +├── pyproject.toml # Project configuration +├── serpapi/ # Package source code +├── tests/ # Test suite +├── examples/ # Usage examples +└── README.md # This file +``` + +## License + +MIT License - see LICENSE file for details. diff --git a/HISTORY.md b/HISTORY.md deleted file mode 100644 index 119737d..0000000 --- a/HISTORY.md +++ /dev/null @@ -1,79 +0,0 @@ -Release History -=============== - -1.2.0 (2026-10-30) ------------------- - -- Add Extract API support through `client.extract()` and `serpapi.extract()` for scraping, URL discovery, and crawling. -- Add Extract API documentation, examples, and tests. -- Add Python 3.15 to the test matrix. - -1.1.2 (2026-09-15) ------------------- - -- Fixed a bug in SerpResults.yield_pages to ensure it does not request more pages than specified by max_pages. The bug - caused two network requests to be sent when max_pages was set to 1. -- Enhancements to test parameters and examples - -1.1.1 (2026-09-08) ------------------- - -- Enhancements for SerpApi markdown output (`output=md`) support -- Enhancements for readthedocs documentation: Added user guide, examples, automated publishing to RTD on release. - -1.1.0 (2026-08-14) ------------------- - -- Add PyPI trusted publishing. PYPI Key is no longer required for publishing the package. -- Add image upload support (`/image` API) for Google Lens -- Release workflow: Add an option to manually trigger the release workflow and skip live tests -- Add more test cases: Locations API, pagination, etc. - -1.0.1 (2026-03-18) ------------------- - -- Fix release workflow: YAML syntax error in smoke test step and remove broken GitHub Packages publish. - -1.0.0 (2026-03-18) ------------------- - -- Automated PyPI release pipeline via GitHub Actions (tag-triggered: test → build → publish → smoke test). -- Modernized packaging to PEP 621 (pyproject.toml), removing legacy setup.py and Pipfile. -- Added Python 3.13 support. - -0.1.6 (2026-02-16) ------------------- - -- Add support for request timeouts. -- Add status and error codes support - https://serpapi.com/api-status-and-error-codes - -0.1.5 (2023-11-01) ------------------- - -- Python 3.12 support. - -0.1.4 (2023-10-11) ------------------- - -- Add README documentation for various engines. - -0.1.3 (2023-10-06) ------------------- - -- Replace deprecated serpapi_pagination.next_link with 'next'. -- Improve documentation: how to use the client directly for pagination searches. - -0.1.2 (2023-10-03) ------------------- - -- Update project status to Production/Stable. - -0.1.1 (2023-10-03) ------------------- - -- Update documentation link to point to Read the Docs. - -0.1.0 (2023-10-03) ------------------- - -- First release on PyPI. diff --git a/LICENSE b/LICENSE index ed10fe2..c7affab 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018-2023 SerpApi +Copyright (c) 2025 SerpApi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 4ddc91b..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,7 +0,0 @@ -include README.md CONTRIBUTING.md HISTORY.md LICENSE -include .readthedocs.yaml -include scripts/check_docs_revision.py scripts/publish_docs.py -recursive-include tests *.py -recursive-include docs *.py *.md *.txt *.css Makefile -include assets/serpapi-logo.svg assets/serpapi-icon.png -prune docs/_build diff --git a/README.md b/README.md index f586dd8..77357e3 100644 --- a/README.md +++ b/README.md @@ -1,462 +1,209 @@ -# SerpApi Python Library & Package -[![Package](https://img.shields.io/pypi/v/serpapi?color=green)](https://pypi.org/project/serpapi) [![serpapi-python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) +# SerpApi Python Library -Integrate search data into your AI workflow, RAG / fine-tuning, or Python application using this official wrapper for [SerpApi](https://serpapi.com). +Official Python client for SerpApi.com - Search Engine Results API. -SerpApi supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and [more](https://serpapi.com). +## Features -Query a vast range of data at scale, including web search results, flight schedules, stock market data, news headlines, and [more](https://serpapi.com). +- Async/await support for non-blocking HTTP requests +- Persistent connections for 2x faster response times +- Multiple search engines (Google, Bing, Yahoo, Baidu, Yandex, etc.) +- Comprehensive API coverage (Search, Location, Account, Search Archive) +- Type hints and extensive documentation ## Installation -To install the `serpapi` package, simply run the following command: - ```bash -$ pip install serpapi -``` - -Please note that this package is separate from the legacy `serpapi` module, which is available on PyPi as `google-search-results`. This package is maintained by SerpApi, and is the recommended way to access the SerpApi service from Python. - -## Quick start - -Let's start by searching for Coffee on Google: +# Using pip +pip install serpapi -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("SERPAPI_KEY")) -results = client.search({ - "engine": "google", - "q": "coffee" -}) - -print(results) +# Using uv (recommended) +uv add serpapi ``` -The `results` variable now contains a `SerpResults` object, which acts just like a standard dictionary, with some convenient functions added on top. - -This example runs a search for "coffee" on Google. It then returns the results as a regular Python Hash. - See the [playground](https://serpapi.com/playground) to generate your own code. - -The SerpApi key can be obtained from [serpapi.com/signup](https://serpapi.com/users/sign_up?plan=free). - -Environment variables are a secure, safe, and easy way to manage secrets. - Set `export SERPAPI_KEY=` in your shell. - Python accesses these variables from `os.environ["SERPAPI_KEY"]`. - -### Extract content from a URL - -The [Extract API](https://serpapi.com/extract-api) supports single-page scraping, URL discovery, and crawling. Call `client.extract()` with a parameter dictionary or keyword arguments. The client sets `engine="extract"` automatically. - -Set `output="md"` to get the page's main content as a Markdown string: +## Quick Start ```python -import os -import serpapi +import asyncio +from serpapi import Client -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=60) -markdown = client.extract(url="https://serpapi.com/", output="md") -print(markdown) -``` - -JSON is the default output format. It returns a dictionary-like `SerpResults` object with page information, structured data, links, and Markdown: +async def main(): + client = Client(api_key="your_api_key", engine="google") + results = await client.search({"q": "coffee"}) + + for result in results.get("organic_results", []): + print(f"Title: {result.get('title')}") + print(f"Link: {result.get('link')}") + + await client.close() -```python -results = client.extract(url="https://serpapi.com/") -print(results["page_information"]["title"]) -print(results["markdown"]) +asyncio.run(main()) ``` -Use `output="html"` to get the retrieved HTML as a string: + → [SerpApi documentation](https://serpapi.com/search-api). -```python -html = client.extract(url="https://serpapi.com/", output="html") -``` +## API Key -Use `mode="map"` to discover URLs or `mode="crawl"` to extract multiple pages: +Get your API key from [serpapi.com/signup](https://serpapi.com/users/sign_up?plan=free). -```python -urls = client.extract(url="https://serpapi.com/", mode="map", include_sitemap=False) -pages = client.extract( - url="https://serpapi.com/", mode="crawl", max_pages=2, max_depth=1, timeout=180 -) +Set environment variable: +```bash +export SERPAPI_KEY="your_secret_key" ``` -Crawl mode can consume one search credit per successfully extracted page, including the starting page. Set `max_pages` to limit the number of pages. - -See the [Extract API guide](https://github.com/serpapi/serpapi-python/blob/master/docs/user_guide/extract.md) for response examples and more options. - -### Error handling +#### Documentations +This library is well documented, and you can find the following resources: + * [Full documentation on SerpApi.com](https://serpapi.com) + * [Library Github page](https://github.com/serpapi/serpapi-ruby) + * [Library GEM page](https://rubygems.org/gems/serpapi/) + * [Library API documentation](https://rubydoc.info/github/serpapi/serpapi-ruby/master) + * [API health status](https://serpapi.com/status) -Unsuccessful requests raise `serpapi.HTTPError` or `serpapi.TimeoutError` exceptions. The returned status code will reflect the sort of error that occurred, please refer to [Status and Error Codes Documentation](https://serpapi.com/api-status-and-error-codes) for more details. +## Usage Examples +### Basic Search ```python -import os -import serpapi - -# A default timeout can be set here. -client = serpapi.Client(api_key=os.getenv("API_KEY"), timeout=10) - -try: - results = client.search({ - 'engine': 'google', - 'q': 'coffee', - }) -except serpapi.HTTPError as e: - if e.status_code == 401: # Invalid API key - print(e.error) # "Invalid API key. Your API key should be here: https://serpapi.com/manage-api-key" - elif e.status_code == 400: # Missing required parameter - pass - elif e.status_code == 429: # Exceeds the hourly throughput limit OR account run out of searches - pass -except serpapi.TimeoutError as e: - # Handle timeout - print(f"The request timed out: {e}") +results = await client.search({"q": "coffee"}) +print(f"Found {len(results.get('organic_results', []))} organic results") ``` -## Documentation - -Documentation is [available on Read the Docs](https://serpapi-python.readthedocs.io/en/latest/). - -Change history is [available on GitHub](https://github.com/serpapi/serpapi-python/blob/master/HISTORY.md). - -### Markdown output for AI agents - -For AI agents and LLM workflows, use `output="md"`. It returns search results with clean headings, links, and tables while using roughly half the tokens of JSON on average. - +### HTML Search ```python -markdown = client.search({ - "engine": "google", - "q": "coffee", - "output": "md", -}) +html_content = await client.html({"q": "coffee"}) +print(f"HTML content length: {len(html_content)} characters") ``` -Markdown works across SerpApi APIs and is returned as a plain Python string. See [Markdown Output for AI Agents](https://serpapi.com/markdown-output.md) for details. - -Raw HTML is also available with `output="html"` and is returned as a plain Python string. - -## Basic examples - -### Search Bing +### Location API ```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'bing', - 'q': 'coffee' -}) +locations = await client.location({"q": "Austin", "limit": 3}) +print(f"Found {len(locations)} locations") ``` -- API Documentation: [serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) -### Search Baidu +### Search Archive ```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'baidu', - 'q': 'coffee', -}) +archived = await client.search_archive(search_id) +print(f"Retrieved archived search: {archived.get('search_metadata', {}).get('id')}") ``` -- API Documentation: [serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) -### Search Yahoo +### Account Info ```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'yahoo', - 'p': 'coffee', -}) +account = await client.account() +print(f"Account plan: {account.get('plan')}") ``` -- API Documentation: [serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) -### Search YouTube -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'youtube', - 'search_query': 'coffee', -}) -``` -- API Documentation: [serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) +## Async Batch Processing -### Search Walmart ```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'walmart', - 'query': 'coffee', -}) -``` -- API Documentation: [serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) +import asyncio -### Search eBay -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'ebay', - '_nkw': 'coffee', -}) -``` -- API Documentation: [serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) +async def search_company(client, company): + results = await client.search({"q": company}) + return {"company": company, "count": len(results.get("organic_results", []))} -### Search Naver -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'naver', - 'query': 'coffee', -}) -``` -- API Documentation: [serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) +async def main(): + client = Client(api_key="your_api_key", persistent=True) + companies = ["meta", "amazon", "apple", "netflix", "google"] + + tasks = [search_company(client, company) for company in companies] + results = await asyncio.gather(*tasks) + + for result in results: + print(f"{result['company']}: {result['count']} results") + + await client.close() -### Search Home Depot -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'home_depot', - 'q': 'chair', -}) +asyncio.run(main()) ``` -- API Documentation: [serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) -### Search Apple App Store -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'apple_app_store', - 'term': 'coffee', -}) -``` -- API Documentation: [serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) +## Context Manager -### Search DuckDuckGo ```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'duckduckgo', - 'q': 'coffee', -}) +async with Client(api_key="your_api_key") as client: + results = await client.search({"q": "coffee"}) + # Client automatically closed ``` -- API Documentation: [serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) -### Search Google -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google', - 'q': 'coffee' -}) -``` -- API Documentation: [serpapi.com/search-api](https://serpapi.com/search-api) +## Error Handling -### Search Google Scholar ```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_scholar', - 'q': 'coffee', -}) -``` -- API Documentation: [serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) +from serpapi import SerpApiError -### Search Google Autocomplete -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_autocomplete', - 'q': 'coffee', -}) +try: + results = await client.search({"q": "coffee"}) +except SerpApiError as e: + print(f"SerpApi error: {e}") ``` -- API Documentation: [serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) -### Search Google Immersive Product +## Developer guide + +The UV Package Manager must be installed. See [uv installation instructions](https://docs.astral.sh/uv/getting-started/installation). -Get a current `page_token` from Google Shopping before requesting product details: +The library also includes a `Rakefile` for developers coming from the Ruby ecosystem, providing a familiar interface for common tasks: -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -shopping = client.search( - engine="google_shopping", - q="lg tv", - gl="us", - hl="en", - no_cache=True, -) -page_token = next( - product["immersive_product_page_token"] - for product in shopping["shopping_results"] - if product.get("immersive_product_page_token") -) -results = client.search( - engine="google_immersive_product", - page_token=page_token, -) -``` -- API Documentation: [serpapi.com/google-immersive-product-api](https://serpapi.com/google-immersive-product-api) - -### Search Google Reverse Image -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_reverse_image', - 'image_url': 'https://upload.wikimedia.org/wikipedia/commons/6/63/Wikipedia-logo.png', - 'max_results': '1', -}) -``` -- API Documentation: [serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) - -### Search Google Lens by image URL or upload +```bash +# Run tests with coverage +rake coverage -Google Lens accepts either a publicly accessible image URL or an uploaded -image. To search by URL, pass the URL directly: +# Run all quality checks (lint, test, coverage) +rake default -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("SERPAPI_KEY")) -results = client.search({ - "engine": "google_lens", - "url": "https://i.imgur.com/HBrB8p0.png", -}) +# Build and test local package +rake oobt ``` -To search a local image, place a file named `image.png` in your working directory, upload it, and pass its temporary `image_id` to Google Lens: +The following commands are also available via `uv`: -```python -upload = client.upload_image("image.png") -results = client.search({ - "engine": "google_lens", - "image_id": upload["image_id"], -}) -``` +```bash +# Install dependencies (including formatting tools) +uv sync --dev +# Run tests +uv run pytest -Uploaded images can be JPG/JPEG, PNG, or WebP files up to 500 KB. The returned -`image_id` expires after 10 minutes. +# Run test with coverage +uv run pytest --cov=serpapi tests/ -- API Documentation: [Google Lens image uploads](https://serpapi.com/google-lens-upload-an-image), [Image API](https://serpapi.com/image-api) +# Type checking with mypy +uv run mypy serpapi/ -### Search Google Local Services -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_local_services', - 'q': 'electrician', - 'data_cid': '6745062158417646970', -}) -``` -- API Documentation: [serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) +# Format code with black +uv run black serpapi/ -### Search Google Maps -```python -import os -import serpapi +# Sort imports with isort +uv run isort serpapi/ - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_maps', - 'q': 'pizza', - 'll': '@40.7455096,-74.0083012,15.1z', - 'type': 'search', -}) +# Check formatting without making changes +uv run black --check . +uv run isort --check-only . ``` -- API Documentation: [serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) -### Search Google Jobs -```python -import os -import serpapi +### UV Benefits +- **Fast**: 10-100x faster than pip +- **Reliable**: Lock file ensures reproducible builds +- **Simple**: Single command for most operations +- **Modern**: Built for Python 3.11+ with async support -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_jobs', - 'q': 'software engineer', -}) -``` -- API Documentation: [serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) +### Project Structure with UV -### Search Google Play -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_play', - 'q': 'kite', - 'store': 'apps', -}) ``` -- API Documentation: [serpapi.com/google-play-api](https://serpapi.com/google-play-api) - -### Search Google Images -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_images', - 'tbm': 'isch', - 'q': 'coffee', -}) +serpapi-python/ +├── .python-version # Python version (3.11) +├── uv.lock # Dependency lock file +├── .venv/ # Virtual environment (auto-created) +├── pyproject.toml # Project configuration +├── serpapi/ # Package source code +├── tests/ # Test suite +├── examples/ # Usage examples +└── README.md # This file ``` -- API Documentation: [serpapi.com/google-images-api](https://serpapi.com/google-images-api) -## License +## Changelog -MIT License. + - 0.2.0: python 3.11+ support only because of dependency changes + - switch to aiohttp as HTTP client + - 3x faster than previous versions + - same API frontend + - 0.1.5: python 3.8 - 3.12 support -## Contributing +## License -See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, testing, and publishing instructions. +MIT License - see LICENSE file for details. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..b37d785 --- /dev/null +++ b/Rakefile @@ -0,0 +1,98 @@ +# Rakefile for serpapi-python +# This file provides a familiar interface for developers coming from the Ruby version. + +require 'rake' + +desc "Execute all the steps (default)" +task default: %i[check dependency version lint test coverage oobt] + +desc "Run out of box testing using the local build" +task oobt: %i[check lint build coverage demo] + +desc "Check if SERPAPI_KEY is set" +task :check do + if ENV['SERPAPI_KEY'] + puts 'check: found $SERPAPI_KEY' + else + puts 'check: SERPAPI_KEY must be defined' + exit 1 + end +end + +desc "Install project dependencies" +task :dependency do + sh 'uv sync --dev' +end + +desc "Print current version" +task :version do + sh 'uv run python -c "from serpapi.version import __version__; print(__version__)"' +end + +desc "Lint code using black, isort, and mypy" +task :lint do + puts "Running black..." + sh 'uv run black --check .' + puts "Running isort..." + sh 'uv run isort --check .' + puts "Running mypy..." + sh 'uv run mypy serpapi/' +end + +desc "Format code using black and isort" +task :format do + sh 'uv run black .' + sh 'uv run isort .' +end + +desc "Run core tests using pytest" +task :test do + sh 'uv run pytest tests/test_client.py' +end + +desc "Run all tests including examples (regression)" +task :regression do + sh 'uv run pytest tests/' +end + +desc "Run tests with coverage" +task :coverage do + sh 'uv run pytest --cov=serpapi tests/' +end + +desc "Build the package (wheel and sdist)" +task :build do + sh 'uv build' +end + +desc "Install the package locally from the build" +task :install do + sh 'uv pip install --force-reinstall dist/*.whl' +end + +desc "Run demo examples" +task :demo do + Dir.glob('examples/*.py').each do |file| + puts "running demo: #{file}" + sh "uv run python #{file}" + end +end + +desc "Run benchmark tests" +task :benchmark do + sh 'uv run python examples/benchmark_serpapi.py' +end + +desc "Create a git tag based on version.py" +task :tag do + version = `uv run python -c "from serpapi.version import __version__; print(__version__)"`.strip + puts "create git tag #{version}" + sh "git tag #{version}" + puts "now publish the tag:\n$ git push origin #{version}" +end + +desc "Release the package (stub for uv publish)" +task release: [:oobt] do + puts "Ready to release. Run: uv publish" +end + diff --git a/assets/serpapi-icon.png b/assets/serpapi-icon.png deleted file mode 100644 index e3754a2..0000000 Binary files a/assets/serpapi-icon.png and /dev/null differ diff --git a/assets/serpapi-logo.svg b/assets/serpapi-logo.svg deleted file mode 100644 index dd3bb9d..0000000 --- a/assets/serpapi-logo.svg +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - SerpApi - diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d4bb2cb..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/custom.css b/docs/_static/custom.css deleted file mode 100644 index f525388..0000000 --- a/docs/_static/custom.css +++ /dev/null @@ -1,99 +0,0 @@ -/* Official palette: https://serpapi.com/media */ -:root { - --serpapi-blue: #377fea; - --serpapi-purple: #6937ea; - --serpapi-dark: #313131; - --serpapi-gradient: linear-gradient(47deg, #377fea, #6937ea); - --sd-color-primary: var(--serpapi-purple); - --sd-color-primary-highlight: var(--serpapi-purple); - --sd-color-primary-text: #ffffff; - --sd-color-secondary: var(--serpapi-blue); - --sd-color-tabs-label-active: var(--serpapi-purple); - --sd-color-tabs-label-hover: var(--serpapi-purple); - --sd-color-tabs-underline-active: var(--serpapi-blue); - --sd-color-tabs-underline-hover: var(--serpapi-blue); -} - -.wy-side-nav-search > a img.logo { - width: 210px; - height: auto; -} - -.wy-side-nav-search, -.wy-nav-top { - background-color: #2b2145; - color: #ffffff; - border-bottom: 4px solid var(--serpapi-blue); - border-image: var(--serpapi-gradient) 1; -} - -.wy-nav-top a { - color: #ffffff; -} - -.wy-side-nav-search input[type="text"] { - border-color: var(--serpapi-blue); -} - -.wy-nav-side { - background: var(--serpapi-dark); -} - -.wy-menu-vertical header, -.wy-menu-vertical p.caption { - color: #ffffff; - height: auto; - min-height: 32px; - line-height: 1.5; - padding-top: 0.4em; - padding-bottom: 0.4em; - white-space: normal; -} - -.wy-menu-vertical a:hover { - background-color: var(--serpapi-purple); - color: #ffffff; -} - -.wy-menu-vertical > ul + ul { - margin-top: 12px; -} - -.wy-menu-vertical > ul + ul > li > a { - font-weight: 700; -} - -.wy-menu-vertical li.current > a, -.wy-menu-vertical li.current > a:hover { - color: var(--serpapi-purple); - box-shadow: inset 4px 0 var(--serpapi-blue); -} - -.wy-nav-content { - color: var(--serpapi-dark); -} - -/* Purple stays readable for small text on the light content background. */ -.wy-nav-content a:not(.btn), -.wy-nav-content a:not(.btn):visited, -.rst-content code.literal, -.rst-content a code { - color: var(--serpapi-purple); -} - -.wy-nav-content a:not(.btn):hover { - text-decoration: underline; -} - -html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt { - color: var(--serpapi-purple); - background: rgba(105, 55, 234, 0.06); - border-top-color: var(--serpapi-blue); -} - -.wy-nav-content a:focus-visible, -.wy-menu-vertical a:focus-visible, -.wy-side-nav-search input[type="text"]:focus-visible { - outline: 2px solid var(--serpapi-blue); - outline-offset: 2px; -} diff --git a/docs/ai-agents.md b/docs/ai-agents.md deleted file mode 100644 index aaf78c1..0000000 --- a/docs/ai-agents.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: "AI Agents" -description: "Use SerpApi search tools, Markdown results, and API references in AI agents." ---- - -# AI Agents - -An AI agent can call SerpApi to search the web and use the returned text and links when answering a question. - -## SerpApi Search Tools - -If you use a Python agent SDK, we recommend [SerpApi Search Tools](https://github.com/serpapi/serpapi-search-tools-python). It is a separate package for plugging SerpApi search tools into your agents. - -## Request Markdown Results - -To write your own search tool with `serpapi.Client`, pass `output="md"`. Markdown is plain text with formatting for headings, links, and tables. - -Set your API key as described in [Getting Started](user_guide/getting-started.md#installation), then run: - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) -markdown = client.search(engine="google", q="Python release news", output="md") -print(markdown[:500]) -``` - -The example prints the first 500 characters. Pass the full `markdown` string back to your agent as the search tool's result. See [Output Formats](user_guide/output-formats.md) for JSON, Markdown, and HTML examples and the Python values they return. - -## API Parameter Reference - -Give your coding agent [SerpApi's `llms.txt`](https://serpapi.com/llms.txt) when asking it to write search code. This file lists the Markdown documentation for each API. Ask the agent to read the relevant API page for required parameters, optional filters, and response fields. diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index c7089ed..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Sphinx configuration for local builds and Read the Docs.""" - -import os - -import serpapi - -project = "serpapi" -copyright = "2026 SerpApi, LLC" -author = "SerpApi, LLC" -release = serpapi.__version__ -version = release - -extensions = ["sphinx.ext.autodoc", "myst_parser", "sphinx_design"] -source_suffix = {".rst": "restructuredtext", ".md": "markdown"} -root_doc = "index" -exclude_patterns = ["_build", "build", "Thumbs.db", ".DS_Store"] -myst_enable_extensions = ["colon_fence"] -myst_heading_anchors = 3 -autodoc_member_order = "bysource" -autodoc_default_options = { - "members": True, - "imported-members": True, - "exclude-members": "HTTPClient, from_http_response", -} - -html_theme = "sphinx_rtd_theme" -html_static_path = ["_static"] -html_css_files = ["custom.css"] -html_logo = "../assets/serpapi-logo.svg" -html_favicon = "../assets/serpapi-icon.png" -html_theme_options = { - "logo_only": True, - "style_nav_header_background": "#2b2145", - "collapse_navigation": False, - "sticky_navigation": True, - "navigation_depth": 3, - "titles_only": True, -} -html_baseurl = os.environ.get( - "READTHEDOCS_CANONICAL_URL", "https://serpapi-python.readthedocs.io/en/latest/" -) diff --git a/docs/examples/amazon-product-search.md b/docs/examples/amazon-product-search.md deleted file mode 100644 index ef767f4..0000000 --- a/docs/examples/amazon-product-search.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: "Amazon Product Search" -description: "Search Amazon products and read prices, ratings, delivery details, and product IDs." ---- - -# Amazon Product Search - -Search Amazon for product listings, prices, ratings, delivery details, and sponsored products. Set `k` to the text you want to search for. - -## Search Products - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="amazon", - k="coffee grinder", - amazon_domain="amazon.com", -) - -for product in results.get("organic_results", [])[:5]: - print(product.get("title")) - print(product.get("price"), product.get("rating"), product.get("reviews")) - print(product.get("asin"), product.get("link_clean")) -``` - -## Read the Results - -Read product listings from `organic_results`. The `asin` field is Amazon's product ID. Useful fields include `title`, `asin`, `price`, `extracted_price`, `rating`, `reviews`, `thumbnail`, `delivery`, `prime`, `link_clean`, and `serpapi_link`. Some searches also return `product_ads`, `featured_products`, `video_results`, `filters`, and `related_searches`. - -For country settings, sorting, product categories, and filters, see the [Amazon Search API documentation](https://serpapi.com/amazon-search-api). Use the [SerpApi Playground](https://serpapi.com/playground) to inspect the response fields for your product category. diff --git a/docs/examples/baidu-search.md b/docs/examples/baidu-search.md deleted file mode 100644 index 8b07f7f..0000000 --- a/docs/examples/baidu-search.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: "Baidu Search" -description: "Search Baidu and read web results for research about China." ---- - -# Baidu Search - -Search Baidu for web pages, for example to research a market in China or check where a website appears in search results. - -## Search Baidu - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="baidu", - q="coffee", -) - -for result in results.get("organic_results", [])[:5]: - print(result.get("position"), result.get("title")) - print(result.get("link")) -``` - -## Read the Results - -Read web listings from `organic_results`. The response may contain other sections depending on the query. Inspect a sample response to see which fields are available before choosing what to store. - -See the [Baidu Search API documentation](https://serpapi.com/baidu-search-api) for supported filters. diff --git a/docs/examples/bing-search.md b/docs/examples/bing-search.md deleted file mode 100644 index 14f8302..0000000 --- a/docs/examples/bing-search.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "Bing Search" -description: "Run a Bing web search and read result titles, links, and positions." ---- - -# Bing Search - -Search Bing to read its web results or compare them with Google results. The example below searches for coffee from Austin, Texas. You can add language, location, and pagination parameters to the same request. - -## Search Bing - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="bing", - q="coffee", - location="Austin, Texas", -) - -for result in results.get("organic_results", [])[:5]: - print(result.get("position"), result.get("title")) - print(result.get("link")) -``` - -## Read the Results - -Read web listings from `organic_results`. Each listing can include a `title`, `link`, `snippet`, and `position`. To see which other sections the response contains, print its keys: - -```python -print(results.keys()) -``` - -For all supported Bing parameters, see the [Bing Search API documentation](https://serpapi.com/bing-search-api). To try different parameters in your browser, use the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/categories/ai-answers.md b/docs/examples/categories/ai-answers.md deleted file mode 100644 index 55e332e..0000000 --- a/docs/examples/categories/ai-answers.md +++ /dev/null @@ -1,7 +0,0 @@ -# AI Answers - -```{toctree} -:maxdepth: 1 - -../google-ai-overview -``` diff --git a/docs/examples/categories/finance-trends-and-jobs.md b/docs/examples/categories/finance-trends-and-jobs.md deleted file mode 100644 index 40e4814..0000000 --- a/docs/examples/categories/finance-trends-and-jobs.md +++ /dev/null @@ -1,10 +0,0 @@ -# Finance, Trends, and Jobs - -```{toctree} -:maxdepth: 1 - -../google-finance-market-data -../google-trends-demand -../google-news-monitoring -../google-jobs-listings -``` diff --git a/docs/examples/categories/local-and-maps.md b/docs/examples/categories/local-and-maps.md deleted file mode 100644 index b701310..0000000 --- a/docs/examples/categories/local-and-maps.md +++ /dev/null @@ -1,8 +0,0 @@ -# Local and Maps - -```{toctree} -:maxdepth: 1 - -../google-maps-local-business -../google-local-services -``` diff --git a/docs/examples/categories/media-apps-and-research.md b/docs/examples/categories/media-apps-and-research.md deleted file mode 100644 index aff4cf0..0000000 --- a/docs/examples/categories/media-apps-and-research.md +++ /dev/null @@ -1,11 +0,0 @@ -# Media, Apps, and Research - -```{toctree} -:maxdepth: 1 - -../youtube-video-search -../google-images-search -../google-lens-image-upload -../google-scholar-research -../google-play-store-apps -``` diff --git a/docs/examples/categories/search-engines.md b/docs/examples/categories/search-engines.md deleted file mode 100644 index 8f2e931..0000000 --- a/docs/examples/categories/search-engines.md +++ /dev/null @@ -1,10 +0,0 @@ -# Search Engines - -```{toctree} -:maxdepth: 1 - -../google-across-countries -../bing-search -../duckduckgo-search -../baidu-search -``` diff --git a/docs/examples/categories/shopping-and-marketplaces.md b/docs/examples/categories/shopping-and-marketplaces.md deleted file mode 100644 index cad4a13..0000000 --- a/docs/examples/categories/shopping-and-marketplaces.md +++ /dev/null @@ -1,12 +0,0 @@ -# Shopping and Marketplaces - -```{toctree} -:maxdepth: 1 - -../google-shopping-products -../google-shopping-price-monitoring -../amazon-product-search -../walmart-product-search -../ebay-product-listings -../home-depot-product-search -``` diff --git a/docs/examples/categories/travel-and-hospitality.md b/docs/examples/categories/travel-and-hospitality.md deleted file mode 100644 index d51a726..0000000 --- a/docs/examples/categories/travel-and-hospitality.md +++ /dev/null @@ -1,8 +0,0 @@ -# Travel and Hospitality - -```{toctree} -:maxdepth: 1 - -../google-flights-travel -../tripadvisor-travel-search -``` diff --git a/docs/examples/duckduckgo-search.md b/docs/examples/duckduckgo-search.md deleted file mode 100644 index 6527419..0000000 --- a/docs/examples/duckduckgo-search.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: "DuckDuckGo Search" -description: "Run a DuckDuckGo search and read web result fields." ---- - -# DuckDuckGo Search - -Search DuckDuckGo to read its web results or compare them with Google and Bing. You can narrow the search with region and time filters. - -## Search DuckDuckGo - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="duckduckgo", - q="coffee", -) - -for result in results.get("organic_results", [])[:5]: - print(result.get("position"), result.get("title")) - print(result.get("link")) -``` - -## Read the Results - -Read web listings from `organic_results`. Each listing can include `title`, `link`, `snippet`, and `position`. Inspect the response for other sections returned by your query. - -See the [DuckDuckGo Search API documentation](https://serpapi.com/duckduckgo-search-api) for region, date, and safe-search options. diff --git a/docs/examples/ebay-product-listings.md b/docs/examples/ebay-product-listings.md deleted file mode 100644 index 49bd417..0000000 --- a/docs/examples/ebay-product-listings.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "eBay Product Listings" -description: "Search eBay listings and read product, price, and seller fields." ---- - -# eBay Product Listings - -Search eBay for listings with prices, item conditions, and seller information. Set `_nkw` to your search text. - -## Search eBay - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="ebay", - _nkw="coffee", -) - -for listing in results.get("organic_results", [])[:5]: - print(listing.get("title")) - print(listing.get("price"), listing.get("condition")) - print(listing.get("link")) -``` - -## Read the Results - -Read the listings from `organic_results`. Useful fields include `title`, `price`, `condition`, `shipping`, `location`, `seller`, `thumbnail`, and `link`. - -See the [eBay Search API documentation](https://serpapi.com/ebay-search-api) for filters, sorting, and pagination. diff --git a/docs/examples/google-across-countries.md b/docs/examples/google-across-countries.md deleted file mode 100644 index a94f184..0000000 --- a/docs/examples/google-across-countries.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -title: "Google Across Countries" -description: "Run the same Google Search across several countries and locations." ---- - -# Google Across Countries - -Compare Google results by setting the country (`gl`), interface language (`hl`), and search location (`location`). Use `google_domain` if you need a particular Google domain. - -## Compare Several Countries - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -markets = [ - {"name": "United States", "gl": "us", "hl": "en", "location": "Austin, Texas"}, - {"name": "United Kingdom", "gl": "gb", "hl": "en", "location": "London, England"}, - {"name": "France", "gl": "fr", "hl": "fr", "location": "Paris, France"}, - {"name": "Germany", "gl": "de", "hl": "de", "location": "Berlin, Germany"}, - {"name": "India", "gl": "in", "hl": "en", "location": "Mumbai, Maharashtra"}, -] - -for market in markets: - results = client.search( - engine="google", - q="best coffee beans", - location=market["location"], - gl=market["gl"], - hl=market["hl"], - ) - - organic = results.get("organic_results", []) - first = organic[0] if organic else {} - - print(market["name"]) - print(first.get("title")) - print(first.get("link")) - print() -``` - -## Use a Google Domain - -Set `google_domain="google.co.in"` to search through Google's Indian domain: - -```python -results = client.search( - engine="google", - q="best coffee beans", - google_domain="google.co.in", - gl="in", - hl="en", - location="Mumbai, Maharashtra", -) -``` - -## Finding Valid Locations - -Call `client.locations()` to find location names accepted by SerpApi: - -```python -locations = client.locations(q="Mumbai", limit=5) - -for location in locations: - print(location.get("canonical_name")) -``` - -See [Account and Locations](../user_guide/account-and-locations.md) for details on choosing a location. Use the [SerpApi Playground](https://serpapi.com/playground) to test location values with the rest of your search parameters. diff --git a/docs/examples/google-ai-overview.md b/docs/examples/google-ai-overview.md deleted file mode 100644 index 2539d54..0000000 --- a/docs/examples/google-ai-overview.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: "Google AI Overview" -description: "Read a Google AI Overview answer or retrieve it with a page token." ---- - -# Google AI Overview - -Some Google searches include an AI Overview answer. If the response contains `ai_overview.page_token`, use that token in a second request to retrieve the answer. Tokens expire quickly, so make the second request as soon as you receive one. - -## Search and Fetch AI Overview - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -search = client.search( - engine="google", - q="how do noise cancelling headphones work", - location="Austin, Texas", - gl="us", - hl="en", -) - -ai_overview = search.get("ai_overview", {}) - -if ai_overview.get("page_token"): - response = client.search( - engine="google_ai_overview", - page_token=ai_overview["page_token"], - ) - ai_overview = response.get("ai_overview", {}) - -for block in ai_overview.get("text_blocks", [])[:3]: - print(block.get("snippet") or block.get("text")) -``` - -## Read the Results - -The `ai_overview` object can contain `text_blocks` with the answer and `references` with its cited sources. If the first Google Search response already includes the AI Overview content, you can read it directly without the second request. - -See the [Google Search API AI Overview docs](https://serpapi.com/search-api#api-examples-results-for-ai-overview) and the [Google AI Overview API documentation](https://serpapi.com/google-ai-overview-api). Use the [SerpApi Playground](https://serpapi.com/playground) to find queries that currently return AI Overview data. diff --git a/docs/examples/google-finance-market-data.md b/docs/examples/google-finance-market-data.md deleted file mode 100644 index ef9140f..0000000 --- a/docs/examples/google-finance-market-data.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "Google Finance Market Data" -description: "Read prices, charts, and related news from Google Finance." ---- - -# Google Finance Market Data - -Look up stocks, indexes, mutual funds, currencies, and futures on Google Finance. You can use the returned prices and charts in a watchlist or dashboard. - -## Fetch a Quote - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_finance", - q="GOOGL:NASDAQ", - window="1M", - hl="en", -) - -summary = results.get("summary", {}) -knowledge = results.get("knowledge_graph", {}) - -print(summary.get("title") or knowledge.get("title")) -print(summary.get("price") or knowledge.get("price")) -print("Graph points:", len(results.get("graph", []))) -``` - -## Read the Results - -Read the name and price from `summary` or `knowledge_graph`, depending on the response. The `graph` list contains values over time, and `news_results` contains related headlines when available. Set `window` to choose the chart's time range. - -See the [Google Finance API documentation](https://serpapi.com/google-finance-api) for supported `q` formats and time windows. You can test symbols in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/google-flights-travel.md b/docs/examples/google-flights-travel.md deleted file mode 100644 index 01d4afa..0000000 --- a/docs/examples/google-flights-travel.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: "Google Flights Travel" -description: "Search for round-trip flights with Google Flights." ---- - -# Google Flights Travel - -Search Google Flights to compare routes, prices, and itineraries. This example searches for a one-week trip starting 30 days from today. `AUS` is Austin and `LAX` is Los Angeles; use the airport codes for your own route. - -## Search Round-Trip Flights - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi -from datetime import date, timedelta - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30) - -results = client.search( - engine="google_flights", - departure_id="AUS", - arrival_id="LAX", - outbound_date=(date.today() + timedelta(days=30)).isoformat(), - return_date=(date.today() + timedelta(days=37)).isoformat(), - currency="USD", - hl="en", - gl="us", -) - -flights = results.get("best_flights") or results.get("other_flights", []) - -for itinerary in flights[:3]: - print("Total price:", itinerary.get("price")) - for flight in itinerary.get("flights", []): - print(flight.get("departure_airport", {}).get("id"), "->", flight.get("arrival_airport", {}).get("id")) -``` - -## Read the Results - -The example reads `best_flights` if it contains results, then tries `other_flights`. Useful fields include `price`, `total_duration`, `carbon_emissions`, and the `flights` list inside each itinerary for airlines, airports, and departure and arrival times. - -For trip type, cabin, date, currency, and airport parameters, see the [Google Flights API documentation](https://serpapi.com/google-flights-api). The [SerpApi Playground](https://serpapi.com/playground) lets you test a route before copying the parameters into Python. diff --git a/docs/examples/google-images-search.md b/docs/examples/google-images-search.md deleted file mode 100644 index f9a8e1d..0000000 --- a/docs/examples/google-images-search.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: "Google Images Search" -description: "Search Google Images and read image titles, sources, and URLs." ---- - -# Google Images Search - -Search Google Images for thumbnails, source pages, and links to the original images. - -## Search Images - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_images", - tbm="isch", - q="coffee", -) - -for image in results.get("images_results", [])[:5]: - print(image.get("title")) - print(image.get("source")) - print(image.get("original")) -``` - -## Read the Results - -Read the images from `images_results`. Useful fields include `title`, `source`, `link`, `thumbnail`, `original`, `original_width`, and `original_height`. - -See the [Google Images API documentation](https://serpapi.com/google-images-api) for image filters and pagination. diff --git a/docs/examples/google-jobs-listings.md b/docs/examples/google-jobs-listings.md deleted file mode 100644 index f2ab6d5..0000000 --- a/docs/examples/google-jobs-listings.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "Google Jobs Listings" -description: "Search job listings by role and location." ---- - -# Google Jobs Listings - -Search Google Jobs for open roles in a location. Results include job titles, employers, and listing sources. You can use them to track hiring or research advertised salaries when salary details are available. - -## Search Jobs - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_jobs", - q="software engineer", - location="Austin, Texas", - hl="en", - gl="us", -) - -for job in results.get("jobs_results", [])[:5]: - print(job.get("title")) - print(job.get("company_name"), "-", job.get("location")) - print(job.get("via")) -``` - -## Read the Results - -Read job listings from `jobs_results`. Typical fields include `title`, `company_name`, `location`, `via`, `description`, `detected_extensions`, and `related_links`. Use the returned links and identifiers to find further details about a listing. - -See the [Google Jobs API documentation](https://serpapi.com/google-jobs-api) for language, location, and pagination parameters. Use the [SerpApi Playground](https://serpapi.com/playground) to try different roles and locations. diff --git a/docs/examples/google-lens-image-upload.md b/docs/examples/google-lens-image-upload.md deleted file mode 100644 index 0f1dcf6..0000000 --- a/docs/examples/google-lens-image-upload.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Google Lens Image Upload" -description: "Upload a local image to SerpApi and search for visual matches with Google Lens." ---- - -# Google Lens Image Upload - -Search Google Lens with an image on your computer. First upload the file with `client.upload_image()`, then pass the returned `image_id` to `client.search()`. - -## Upload and Search - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. Place an image named `image.png` in the directory where you run the script, or replace `image.png` with the path to your file. - -Uploads can be JPG/JPEG, PNG, or WebP files up to 500 KB. The returned image ID expires after ten minutes, so search soon after uploading. See the [Image API documentation](https://serpapi.com/image-api) for upload requirements. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30) - -upload = client.upload_image("image.png") - -results = client.search( - engine="google_lens", - image_id=upload["image_id"], - type="visual_matches", - hl="en", -) - -for match in results.get("visual_matches", [])[:5]: - print(match.get("title")) - print(match.get("source")) - print(match.get("link")) -``` - -`upload_image()` returns a dictionary containing `image_id`. The second call uses that ID to search Google Lens. You can also pass a file opened in binary mode to `upload_image()`. See {py:meth}`serpapi.Client.upload_image` in the API reference. - -## Read the Results - -Read matching images and pages from `visual_matches`. Each match can include a `title`, `source`, `link`, and `thumbnail`. The example prints up to five matches and prints nothing if that list is empty. - -Set `type="products"` to look for products, or `type="exact_matches"` to find pages containing the same image. See the [Google Lens API documentation](https://serpapi.com/google-lens-api) for search types and other parameters. diff --git a/docs/examples/google-local-services.md b/docs/examples/google-local-services.md deleted file mode 100644 index c82a1c6..0000000 --- a/docs/examples/google-local-services.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: "Google Local Services" -description: "Read Google Local Services advertisements for a service category." ---- - -# Google Local Services - -Search Google Local Services for advertisements from local providers, such as electricians. Results include ratings and phone numbers when available. Use the Google Maps API for general place listings. - -## Search Local Services - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_local_services", - q="electrician", - data_cid="6745062158417646970", -) - -for ad in results.get("local_ads", [])[:5]: - print(ad.get("title")) - print(ad.get("rating"), ad.get("reviews")) - print(ad.get("phone")) -``` - -## Read the Results - -Read the advertisements from `local_ads`. Useful fields include `title`, `rating`, `reviews`, `phone`, `service_area`, `years_in_business`, and `link` when present. - -See the [Google Local Services API documentation](https://serpapi.com/google-local-services-api) for query and provider detail options. diff --git a/docs/examples/google-maps-local-business.md b/docs/examples/google-maps-local-business.md deleted file mode 100644 index b227ea7..0000000 --- a/docs/examples/google-maps-local-business.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: "Google Maps Local Businesses" -description: "Find local businesses with Google Maps." ---- - -# Google Maps Local Businesses - -Search Google Maps for nearby businesses, for example to build a store locator, find potential customers, or check local search rankings. Set `q` to the business type or name. The `ll` value specifies the map's latitude, longitude, and zoom level. - -## Search for Local Businesses - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_maps", - q="coffee shops", - ll="@30.2672,-97.7431,14z", - type="search", -) - -for place in results.get("local_results", [])[:5]: - print(place.get("title")) - print(place.get("rating"), place.get("reviews")) - print(place.get("address")) -``` - -## Read the Results - -Read business listings from `local_results`. Common fields include `title`, `rating`, `reviews`, `address`, `phone`, `website`, and `gps_coordinates`. Some responses also include `place_results` when the query matches a specific place. - -For the complete parameter list, use the [Google Maps API documentation](https://serpapi.com/google-maps-api). Use the [SerpApi Playground](https://serpapi.com/playground) to choose an `ll` value for the area you want to search. diff --git a/docs/examples/google-news-monitoring.md b/docs/examples/google-news-monitoring.md deleted file mode 100644 index 63c1dfd..0000000 --- a/docs/examples/google-news-monitoring.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: "Google News Monitoring" -description: "Collect recent headlines for a topic with Google News." ---- - -# Google News Monitoring - -Search Google News for articles about a topic. You can collect headlines over time or use new matches to trigger alerts. - -## Search News - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_news", - q="artificial intelligence", - gl="us", - hl="en", -) - -for item in results.get("news_results", [])[:5]: - print(item.get("title")) - print(item.get("source", {}).get("name") or item.get("source")) - print(item.get("link")) -``` - -## Read the Results - -Read articles from `news_results`. Save `title`, `link`, `source`, `date`, `snippet`, and `thumbnail` when present. Response fields vary by query, so inspect a sample response before choosing which fields to store. - -See the [Google News API documentation](https://serpapi.com/google-news-api) and try your topic in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/google-play-store-apps.md b/docs/examples/google-play-store-apps.md deleted file mode 100644 index f60fc35..0000000 --- a/docs/examples/google-play-store-apps.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: "Google Play Store Apps" -description: "Search Google Play for apps and read their titles, ratings, and links." ---- - -# Google Play Store Apps - -Search Google Play for apps and read their ratings, descriptions, and store links. - -## Search Apps - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_play", - q="kite", - store="apps", - max_results="2", -) - -for app in results.get("organic_results", [])[:5]: - print(app.get("title")) - print(app.get("rating"), app.get("downloads")) - print(app.get("link")) -``` - -## Read the Results - -Read app listings from `organic_results`. Useful fields include `title`, `link`, `rating`, `downloads`, `description`, `thumbnail`, and app identifiers. - -See the [Google Play Store API documentation](https://serpapi.com/google-play-api) for store, device, and localization options. diff --git a/docs/examples/google-scholar-research.md b/docs/examples/google-scholar-research.md deleted file mode 100644 index 22b78e0..0000000 --- a/docs/examples/google-scholar-research.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "Google Scholar Research" -description: "Search Google Scholar for publications and citation counts." ---- - -# Google Scholar Research - -Search Google Scholar for publications, citation counts, and author links. Repeat a query to check for new research on a topic. - -## Search Scholar - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_scholar", - q="coffee", -) - -for result in results.get("organic_results", [])[:5]: - print(result.get("title")) - print(result.get("publication_info", {}).get("summary")) - print(result.get("inline_links", {}).get("cited_by", {}).get("total")) -``` - -## Read the Results - -Read publications from `organic_results`. Useful fields include `title`, `link`, `publication_info`, `snippet`, `resources`, and `inline_links.cited_by`. - -See the [Google Scholar API documentation](https://serpapi.com/google-scholar-api) for author, citation, and date filters. diff --git a/docs/examples/google-shopping-price-monitoring.md b/docs/examples/google-shopping-price-monitoring.md deleted file mode 100644 index e40a6e9..0000000 --- a/docs/examples/google-shopping-price-monitoring.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: "Google Shopping Price Monitoring" -description: "Compare US espresso-machine listings within a budget." ---- - -# Google Shopping Price Monitoring - -Search Google Shopping for a De'Longhi Stilosa espresso machine in Austin, Texas, and compare offers priced from $100 to $200. The example filters the returned prices in Python using the numeric `extracted_price` field. - -## Search Filtered Products - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_shopping", - q="DeLonghi Stilosa espresso machine", - location="Austin, Texas", - gl="us", - hl="en", - json_restrictor="error, shopping_results[].{title, price, extracted_price, source, rating, reviews, product_link}", -) - -for product in results.get("shopping_results", []): - price = product.get("extracted_price") - if price is None or not 100 <= price <= 200: - continue - print(product.get("title")) - print(product.get("price"), product.get("source")) - print(product.get("rating"), product.get("reviews")) -``` - -## Read the Results - -Read product listings from `shopping_results`. Useful fields include `title`, `product_id`, `price`, `extracted_price`, `source`, `rating`, `reviews`, `thumbnail`, `delivery`, `product_link`, and `serpapi_immersive_product_api`. - -The example limits response fields with `json_restrictor` and keeps `error` so API errors remain visible. Use `extracted_price` for numeric comparisons and `price` for display. See the [Google Shopping API documentation](https://serpapi.com/google-shopping-api) for the full parameter set. diff --git a/docs/examples/google-shopping-products.md b/docs/examples/google-shopping-products.md deleted file mode 100644 index 061a228..0000000 --- a/docs/examples/google-shopping-products.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "Google Shopping Products" -description: "Search Google Shopping and read product listings." ---- - -# Google Shopping Products - -Search Google Shopping for products, prices, merchants, and product links. Use the results to compare prices, research products, or add merchant information to a product catalog. - -## Search Products - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_shopping", - q="espresso machine", - location="Austin, Texas", - gl="us", - hl="en", -) - -for product in results.get("shopping_results", [])[:5]: - print(product.get("title")) - print(product.get("price"), product.get("source")) - print(product.get("link")) -``` - -## Read the Results - -Read product listings from `shopping_results`. Save `title`, `price`, `source`, `rating`, `reviews`, `thumbnail`, and `link` if present. Check the API documentation for supported filters and sorting options before adding them to your request. - -See the [Google Shopping API documentation](https://serpapi.com/google-shopping-api) and experiment in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/google-trends-demand.md b/docs/examples/google-trends-demand.md deleted file mode 100644 index 5851d4f..0000000 --- a/docs/examples/google-trends-demand.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: "Google Trends Demand" -description: "Compare search interest over time with Google Trends." ---- - -# Google Trends Demand - -Use Google Trends to compare search interest over time and between regions. You can look for seasonal patterns when planning content, researching products, or choosing a launch date. - -## Interest Over Time - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google_trends", - q="electric bikes", - date="today 12-m", - geo="US", - data_type="TIMESERIES", - tz="420", -) - -timeline = results.get("interest_over_time", {}).get("timeline_data", []) - -for point in timeline[-5:]: - value = point.get("values", [{}])[0].get("extracted_value") - print(point.get("date"), value) -``` - -## Read the Results - -Read `interest_over_time.timeline_data` for search interest over time. Set `data_type="GEO_MAP_0"` to compare regions, or use `RELATED_QUERIES` or `RELATED_TOPICS` to find related searches. - -See the [Google Trends API documentation](https://serpapi.com/google-trends-api) for `data_type`, `geo`, `date`, and `tz` options. The [SerpApi Playground](https://serpapi.com/playground) helps confirm that a trend query returns the chart you expect. diff --git a/docs/examples/home-depot-product-search.md b/docs/examples/home-depot-product-search.md deleted file mode 100644 index edaeda2..0000000 --- a/docs/examples/home-depot-product-search.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "Home Depot Product Search" -description: "Search Home Depot products and read prices, ratings, and product IDs." ---- - -# Home Depot Product Search - -Search The Home Depot for home improvement products. You can use the listings to track prices or add product details to a catalog. Inspect the response for your query before adding category filters. - -## Search Products - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="home_depot", - q="chair", -) - -for product in results.get("products", [])[:5]: - print(product.get("title")) - print(product.get("price"), product.get("rating")) - print(product.get("product_id")) -``` - -## Read the Results - -Read product listings from `products`. Useful fields include `title`, `product_id`, `price`, `rating`, `reviews`, `brand`, `thumbnail`, and product URLs when present. - -See [The Home Depot Search API documentation](https://serpapi.com/home-depot-search-api) for product filters and pagination. diff --git a/docs/examples/index.md b/docs/examples/index.md deleted file mode 100644 index 4d2ccaf..0000000 --- a/docs/examples/index.md +++ /dev/null @@ -1,205 +0,0 @@ ---- -title: "Examples" -description: "Python search examples grouped by API category." ---- - -# Examples - -Each example makes a search and prints selected fields from the response. Before running one, [install the package and set your API key](../user_guide/getting-started.md#installation). - -The examples use `results.get("organic_results", [])` or a similar expression to read a list of results. The empty list `[]` is the default if that section is missing. Within a result, `item.get("title")` returns `None` if the title is missing. A slice such as `[:5]` limits the printed list to its first five items. - -Follow the API documentation linked from each example for all supported parameters. The API categories below are also listed in [SerpApi's `llms.txt`](https://serpapi.com/llms.txt). - -## Starter Examples - -### Search Engines - -- [Google Across Countries](google-across-countries.md) -- [Bing Search](bing-search.md) -- [DuckDuckGo Search](duckduckgo-search.md) -- [Baidu Search](baidu-search.md) - -### AI Answers - -- [Google AI Overview](google-ai-overview.md) - -### Local and Maps - -- [Google Maps Local Businesses](google-maps-local-business.md) -- [Google Local Services](google-local-services.md) - -### Shopping and Marketplaces - -- [Google Shopping Products](google-shopping-products.md) -- [Google Shopping Price Monitoring](google-shopping-price-monitoring.md) -- [Amazon Product Search](amazon-product-search.md) -- [Walmart Product Search](walmart-product-search.md) -- [eBay Product Listings](ebay-product-listings.md) -- [Home Depot Product Search](home-depot-product-search.md) - -### Travel and Hospitality - -- [Google Flights Travel](google-flights-travel.md) -- [Tripadvisor Travel Search](tripadvisor-travel-search.md) - -### Finance, Trends, and Jobs - -- [Google Finance Market Data](google-finance-market-data.md) -- [Google Trends Demand](google-trends-demand.md) -- [Google News Monitoring](google-news-monitoring.md) -- [Google Jobs Listings](google-jobs-listings.md) - -### Media, Apps, and Research - -- [YouTube Video Search](youtube-video-search.md) -- [Google Images Search](google-images-search.md) -- [Google Lens Image Upload](google-lens-image-upload.md) -- [Google Scholar Research](google-scholar-research.md) -- [Google Play Store Apps](google-play-store-apps.md) - -## Supported API Map - -**Search Engines** - -- [Google Search API](https://serpapi.com/search-api) -- [Google Light Search API](https://serpapi.com/google-light-api) -- [Bing Search API](https://serpapi.com/bing-search-api) -- [DuckDuckGo Search API](https://serpapi.com/duckduckgo-search-api) -- [DuckDuckGo Light API](https://serpapi.com/duckduckgo-light-api) -- [Baidu Search API](https://serpapi.com/baidu-search-api) -- [Yahoo! Search API](https://serpapi.com/yahoo-search-api) -- [Yandex Search API](https://serpapi.com/yandex-search-api) -- [Naver Search API](https://serpapi.com/naver-search-api) - -**AI Answers** - -- [Google AI Mode API](https://serpapi.com/google-ai-mode-api) -- [Google AI Overview API](https://serpapi.com/google-ai-overview-api) -- [Bing Copilot API](https://serpapi.com/bing-copilot-api) -- [Brave AI Mode API](https://serpapi.com/brave-ai-mode-api) -- [Naver AI Overview API](https://serpapi.com/naver-ai-overview-api) - -**Local and Maps** - -- [Google Local API](https://serpapi.com/google-local-api) -- [Google Local Services API](https://serpapi.com/google-local-services-api) -- [Google Maps API](https://serpapi.com/google-maps-api) -- [Google Maps Photos API](https://serpapi.com/google-maps-photos-api) -- [Google Maps Autocomplete API](https://serpapi.com/google-maps-autocomplete-api) -- [Google Maps Directions API](https://serpapi.com/google-maps-directions-api) -- [Google Maps Posts API](https://serpapi.com/google-maps-posts-api) -- [Google Maps Reviews API](https://serpapi.com/google-maps-reviews-api) -- [Google Maps Contributor Reviews API](https://serpapi.com/google-maps-contributor-reviews-api) -- [Apple Maps API](https://serpapi.com/apple-maps-api) -- [Apple Maps Places API](https://serpapi.com/apple-maps-places-api) -- [Bing Maps API](https://serpapi.com/bing-maps-api) -- [DuckDuckGo Maps API](https://serpapi.com/duckduckgo-maps-api) -- [Yelp Search API](https://serpapi.com/yelp-search-api) -- [Yelp Place API](https://serpapi.com/yelp-place) -- [Yelp Reviews API](https://serpapi.com/yelp-reviews-api) - -**Shopping and Marketplaces** - -- [Google Shopping API](https://serpapi.com/google-shopping-api) -- [Google Shopping Light API](https://serpapi.com/google-shopping-light-api) -- [Google Immersive Product API](https://serpapi.com/google-immersive-product-api) -- [Amazon Search API](https://serpapi.com/amazon-search-api) -- [Amazon Product API](https://serpapi.com/amazon-product-api) -- [Walmart Search API](https://serpapi.com/walmart-search-api) -- [Walmart Product API](https://serpapi.com/walmart-product-api) -- [Walmart Reviews API](https://serpapi.com/walmart-product-reviews-api) -- [eBay Search API](https://serpapi.com/ebay-search-api) -- [eBay Product API](https://serpapi.com/ebay-product-api) -- [The Home Depot Search API](https://serpapi.com/home-depot-search-api) -- [The Home Depot Product API](https://serpapi.com/home-depot-product) -- [The Home Depot Reviews API](https://serpapi.com/home-depot-product-reviews) -- [Bing Shopping API](https://serpapi.com/bing-shopping-api) -- [Bing Product API](https://serpapi.com/bing-product-api) -- [Yahoo! Shopping API](https://serpapi.com/yahoo-shopping-search-api) - -**Travel and Hospitality** - -- [Google Flights API](https://serpapi.com/google-flights-api) -- [Google Flights Autocomplete API](https://serpapi.com/google-flights-autocomplete-api) -- [Google Flights Deals API](https://serpapi.com/google-flights-deals-api) -- [Google Hotels API](https://serpapi.com/google-hotels-api) -- [Google Hotels Autocomplete API](https://serpapi.com/google-hotels-autocomplete-api) -- [Google Hotels Photos API](https://serpapi.com/google-hotels-photos-api) -- [Google Hotels Reviews API](https://serpapi.com/google-hotels-reviews-api) -- [Google Travel Explore API](https://serpapi.com/google-travel-explore-api) -- [Tripadvisor Search API](https://serpapi.com/tripadvisor-search-api) -- [Tripadvisor Place API](https://serpapi.com/tripadvisor-place-api) -- [Tripadvisor Reviews API](https://serpapi.com/tripadvisor-reviews-api) -- [OpenTable Reviews API](https://serpapi.com/open-table-reviews-api) - -**News, Trends, and Demand** - -- [Google News API](https://serpapi.com/google-news-api) -- [Google News Light API](https://serpapi.com/google-news-light-api) -- [Bing News API](https://serpapi.com/bing-news-api) -- [DuckDuckGo News API](https://serpapi.com/duckduckgo-news-api) -- [Baidu News API](https://serpapi.com/baidu-news-api) -- [Google Trends API](https://serpapi.com/google-trends-api) -- [Google Trends Autocomplete API](https://serpapi.com/google-trends-autocomplete) -- [Google Trends Trending Now API](https://serpapi.com/google-trends-trending-now) -- [Google Sports API](https://serpapi.com/google-sports-api) -- [Google Forums API](https://serpapi.com/google-forums-api) - -**Media, Apps, and Social** - -- [Google Images API](https://serpapi.com/google-images-api) -- [Google Images Light API](https://serpapi.com/google-images-light-api) -- [Google Lens API](https://serpapi.com/google-lens-api) -- [Google Reverse Image API](https://serpapi.com/google-reverse-image) -- [Google Videos API](https://serpapi.com/google-videos-api) -- [Google Videos Light API](https://serpapi.com/google-videos-light-api) -- [Google Short Videos API](https://serpapi.com/google-short-videos-api) -- [Bing Images API](https://serpapi.com/bing-images-api) -- [Bing Reverse Image API](https://serpapi.com/bing-reverse-image-api) -- [Bing Videos API](https://serpapi.com/bing-videos-api) -- [Yahoo! Images API](https://serpapi.com/yahoo-images-api) -- [Yahoo! Videos API](https://serpapi.com/yahoo-videos-api) -- [Yandex Images API](https://serpapi.com/yandex-images-api) -- [Yandex Videos API](https://serpapi.com/yandex-videos-api) -- [YouTube Search API](https://serpapi.com/youtube-search-api) -- [YouTube Video API](https://serpapi.com/youtube-video-api) -- [YouTube Video Transcript API](https://serpapi.com/youtube-video-transcript) -- [Apple App Store API](https://serpapi.com/apple-app-store) -- [Apple App Store Reviews API](https://serpapi.com/apple-reviews) -- [Apple App Store Product API](https://serpapi.com/apple-product) -- [Google Play Store API](https://serpapi.com/google-play-api) -- [Google Play Games API](https://serpapi.com/google-play-games) -- [Google Play Movies API](https://serpapi.com/google-play-movies) -- [Google Play Books API](https://serpapi.com/google-play-books) -- [Google Play Product API](https://serpapi.com/google-play-product-api) -- [Facebook Profile API](https://serpapi.com/facebook-profile-api) -- [Instagram Profile API](https://serpapi.com/instagram-profile-api) - -**Research, Knowledge, Ads, Finance, and Jobs** - -- [Google Scholar API](https://serpapi.com/google-scholar-api) -- [Google Scholar Author API](https://serpapi.com/google-scholar-author-api) -- [Google Scholar Case Law API](https://serpapi.com/google-scholar-case-law-api) -- [Google Patents API](https://serpapi.com/google-patents-api) -- [Google Patents Details API](https://serpapi.com/google-patents-details-api) -- [Google Related Questions API](https://serpapi.com/google-related-questions-api) -- [Google Autocomplete API](https://serpapi.com/google-autocomplete-api) -- [Google Ads API](https://serpapi.com/google-ads-api) -- [Google Ads Transparency API](https://serpapi.com/google-ads-transparency-center-api) -- [Google Finance API](https://serpapi.com/google-finance-api) -- [Google Finance Markets API](https://serpapi.com/google-finance-markets) -- [Google Jobs API](https://serpapi.com/google-jobs-api) - -```{toctree} -:hidden: -:maxdepth: 2 - -categories/search-engines -categories/ai-answers -categories/local-and-maps -categories/shopping-and-marketplaces -categories/travel-and-hospitality -categories/finance-trends-and-jobs -categories/media-apps-and-research -``` diff --git a/docs/examples/tripadvisor-travel-search.md b/docs/examples/tripadvisor-travel-search.md deleted file mode 100644 index 8bccc4a..0000000 --- a/docs/examples/tripadvisor-travel-search.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: "Tripadvisor Travel Search" -description: "Search Tripadvisor for destinations, hotels, restaurants, and attractions." ---- - -# Tripadvisor Travel Search - -Search Tripadvisor for destinations, hotels, restaurants, attractions, and forum posts. Set `ssrc` to limit the search to one type of result, such as hotels. - -## Search Places - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="tripadvisor", - q="Rome", - ssrc="h", -) - -for place in results.get("places", [])[:5]: - print(place.get("title")) - print(place.get("place_type"), place.get("location")) - print(place.get("link")) -``` - -## Read the Results - -Read the returned places from `places`. Useful fields include `title`, `place_type`, `place_id`, `location`, `description`, `thumbnail`, `link`, and `serpapi_link`. - -Choose the result type with `ssrc`: `h` for hotels, `r` for restaurants, `A` for things to do, `g` for destinations, and `a` for all results. See the [Tripadvisor Search API documentation](https://serpapi.com/tripadvisor-search-api) for the current filter and pagination options. diff --git a/docs/examples/walmart-product-search.md b/docs/examples/walmart-product-search.md deleted file mode 100644 index fdfdb48..0000000 --- a/docs/examples/walmart-product-search.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: "Walmart Product Search" -description: "Search Walmart products and read prices, reviews, shipping details, and product IDs." ---- - -# Walmart Product Search - -Search Walmart for products, prices, and availability. You can use the results to track prices or add details to a product catalog. Set `query` to your search text. This engine does not use `q`. - -## Search Products - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="walmart", - query="coffee maker", - walmart_domain="walmart.com", -) - -for product in results.get("organic_results", [])[:5]: - offer = product.get("primary_offer", {}) - print(product.get("title")) - print(offer.get("offer_price"), product.get("rating"), product.get("reviews")) - print(product.get("us_item_id"), product.get("product_page_url")) -``` - -## Read the Results - -Read product listings from `organic_results`. Useful fields include `title`, `us_item_id`, `product_id`, `rating`, `reviews`, `seller_name`, `primary_offer.offer_price`, `price_per_unit`, `out_of_stock`, `product_page_url`, and `serpapi_product_page_url`. - -Use `sort`, `min_price`, `max_price`, `store_id`, and `facet` to sort results, set price limits, or filter by store and product features. See the [Walmart Search API documentation](https://serpapi.com/walmart-search-api) for supported filters and pagination behavior. diff --git a/docs/examples/youtube-video-search.md b/docs/examples/youtube-video-search.md deleted file mode 100644 index ce8d5e7..0000000 --- a/docs/examples/youtube-video-search.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "YouTube Video Search" -description: "Search YouTube and read video titles, channels, and links." ---- - -# YouTube Video Search - -Search YouTube for videos about a topic and read their channel details. Set `search_query` to your search text. This engine does not use `q`. - -## Search Videos - -[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. - -```python -import os -import serpapi - - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="youtube", - search_query="coffee brewing guide", -) - -for video in results.get("video_results", [])[:5]: - print(video.get("title")) - print(video.get("channel", {}).get("name")) - print(video.get("link")) -``` - -## Read the Results - -Read videos from `video_results`. Useful fields include `title`, `link`, `channel`, `views`, `published_date`, `length`, and `thumbnail`. Some queries return additional sections such as channels, playlists, shorts, or related searches. - -See the [YouTube Search API documentation](https://serpapi.com/youtube-search-api) for YouTube-specific parameters. The [SerpApi Playground](https://serpapi.com/playground) lets you inspect the response fields for your query. diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index f416f44..0000000 --- a/docs/index.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -title: "SerpApi Python Library & Package" -description: "Search Google and other engines from Python with the official SerpApi client." ---- - -# SerpApi Python Library & Package - -`serpapi` is the official Python client for [SerpApi](https://serpapi.com). Use it to search the web and read the results in your Python programs. - -SerpApi supports Google, Google Maps, Google Shopping, Bing, DuckDuckGo, Baidu, Yandex, Yahoo, eBay, YouTube, App Stores, Walmart, Home Depot, Naver, and many more engines. - -You can retrieve web search results, local business listings, shopping results, flight schedules, stock market data, job listings, trends, news headlines, AI Overview answers, and video search results. - -## Install - -Run one of these commands in your terminal. Use `pip` to install into your Python environment, or `uv add` if you manage your project with uv. - -::::{tab-set} - -:::{tab-item} pip - -```bash -pip install serpapi -``` - -::: - -:::{tab-item} uv - -```bash -uv add serpapi -``` - -::: - -:::: - -The package requires Python 3.6 or newer. - -## First Request - -Sign up at [SerpApi](https://serpapi.com/users/sign_up) and copy your API key from the [dashboard](https://serpapi.com/manage-api-key). Replace `secret_api_key` below with your key, then run the command in your terminal. On Windows, use PowerShell. - -::::{tab-set} - -:::{tab-item} macOS / Linux - -```bash -export SERPAPI_KEY="secret_api_key" -``` - -::: - -:::{tab-item} Windows - -```powershell -$env:SERPAPI_KEY = "secret_api_key" -``` - -::: - -:::: - -Run this code in a Python script or interpreter started from the same terminal so it can read `SERPAPI_KEY`: - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - hl="en", - gl="us", -) - -print(results["organic_results"][0]["link"]) -``` - -This prints the link from the first organic result, which is an unpaid search listing. `results["organic_results"]` is a list, and `[0]` selects its first item. See [Getting Started](user_guide/getting-started.md) for setup instructions and an explanation of each parameter. - -The `results` variable contains a `SerpResults` object. You can read its fields like a Python dictionary, convert it to a plain dictionary, or use its methods to fetch more pages. You can also retrieve saved searches with `client.search_archive()`. - -Use the same search parameter names as the [SerpApi API documentation](https://serpapi.com/search-api). The [SerpApi Playground](https://serpapi.com/playground) lets you try a search in your browser and copy its parameters into Python. - -## Documentation Map - -- [Getting Started](user_guide/getting-started.md) covers installation and your first search. [Client Usage](user_guide/client-usage.md) explains how to make requests and read responses. -- [AI Agents](ai-agents.md) covers search tools, Markdown results, and API references for agents. -- [Output Formats](user_guide/output-formats.md) explains when to use JSON, Markdown, or HTML. -- [Extract API](user_guide/extract.md) covers page content in Markdown or JSON, URL discovery, and crawling. -- [Migration Guide](user_guide/migrating-from-google-search-results.md) shows how to replace `google-search-results` with this package. -- [Parameters and Engines](user_guide/parameters-and-engines.md) explains search parameters and how to test them in the [SerpApi Playground](https://serpapi.com/playground). -- For scripts that collect results, see [pagination](user_guide/pagination.md), [timeouts and errors](user_guide/errors-and-timeouts.md), [request options](user_guide/request-options.md), and [account and locations](user_guide/account-and-locations.md). -- For searches you retrieve later, selected response fields, and data retention settings, see [Async Search Archive](user_guide/async-search-archive.md), [JSON Restrictor](user_guide/json-restrictor.md), and [Zero Trace](user_guide/zero-trace.md). -- [Examples](examples/index.md) includes searches for web pages, AI answers, local businesses, products, travel, finance, trends, jobs, media, apps, and research. - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: Basics - -user_guide/getting-started -user_guide/client-usage -``` - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: User Guide - -User Guide Overview -user_guide/parameters-and-engines -user_guide/output-formats -user_guide/extract -user_guide/pagination -user_guide/errors-and-timeouts -user_guide/account-and-locations -``` - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: Migration - -user_guide/migrating-from-google-search-results -``` - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: Advanced Usage - -user_guide/async-search-archive -user_guide/threading -user_guide/multiprocessing -user_guide/zero-trace -user_guide/json-restrictor -user_guide/request-options -``` - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: AI Agents - -ai-agents -``` - -```{toctree} -:hidden: -:maxdepth: 3 - -examples/index -``` - -```{toctree} -:hidden: -:maxdepth: 2 -:caption: Reference - -reference -``` diff --git a/docs/reference.md b/docs/reference.md deleted file mode 100644 index 9fbd9ce..0000000 --- a/docs/reference.md +++ /dev/null @@ -1,8 +0,0 @@ -# API Reference - -Look up the arguments, return values, and exceptions for the functions and classes in `serpapi`. For examples of how to use them together, see [Client Usage](user_guide/client-usage.md). - -```{eval-rst} -.. automodule:: serpapi - :member-order: groupwise -``` diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 142b6ca..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e .[docs] diff --git a/docs/user_guide/account-and-locations.md b/docs/user_guide/account-and-locations.md deleted file mode 100644 index d48aab2..0000000 --- a/docs/user_guide/account-and-locations.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: "Account and Locations" -description: "Check your search allowance and find locations for Google searches." ---- - -# Account and Locations - -Use `client.account()` to check your account and `client.locations()` to find locations for a Google search. - -## Account - -After [setting your API key](getting-started.md#installation), call `account()` to read your account information: - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) - -account = client.account() -print(account) -``` - -The returned dictionary includes `total_searches_left`, `this_month_usage`, and `account_rate_limit_per_hour`. You can check these before running a batch of searches. See the [Account API documentation](https://serpapi.com/account-api) for all fields. - -## Locations - -Use `locations()` to find supported Google locations: - -```python -locations = client.locations(q="Austin", limit=5) - -for location in locations: - print(location.get("name"), location.get("canonical_name")) -``` - -Choose a `canonical_name` from the returned locations and pass it to a search. For example: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas, United States", -) -``` - -See the [SerpApi Locations API documentation](https://serpapi.com/locations-api) for supported parameters and response fields. diff --git a/docs/user_guide/async-search-archive.md b/docs/user_guide/async-search-archive.md deleted file mode 100644 index f669653..0000000 --- a/docs/user_guide/async-search-archive.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: "Async Search Archive" -description: "Submit searches now and retrieve their results later with the Search Archive API." ---- - -# Async Search Archive - -An asynchronous search lets you submit a request now and collect the results later. Set the API's `async` parameter to `true` to receive a search ID, then use that ID with the Search Archive API. Each Python client call still waits for an HTTP response. These methods do not use Python's `asyncio` or `await`. - -Create a client as shown in [Client Usage](client-usage.md#create-a-client) before running the examples. - -## Submit an Async Search - -`async` is a reserved word in Python. Put it in a dictionary and use `**` to pass that dictionary's entries as keyword arguments: - -```python -submitted = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - **{"async": True}, -) - -search_id = submitted["search_metadata"]["id"] -print(search_id) -``` - -## Retrieve the Search - -```python -archived = client.search_archive(search_id=search_id) - -status = archived.get("search_metadata", {}).get("status") -print(status) -``` - -Check the status before reading the results. If the search is still processing, wait and call `search_archive()` again. This loop checks every two seconds until the search succeeds or reports an error: - -```python -import time - -while True: - archived = client.search_archive(search_id=search_id) - status = archived.get("search_metadata", {}).get("status") - - if status == "Success": - break - - if status == "Error": - raise RuntimeError(archived.get("error", "Search failed")) - - time.sleep(2) -``` - -## When to Use Async Searches - -Use asynchronous searches when one part of your program submits searches and another collects the results later. For example, a scheduled job could submit a batch of searches for another worker to process. - -Do not combine `async=true` with `no_cache=true`. See the [SerpApi Search Archive API documentation](https://serpapi.com/search-archive-api) for search statuses and how long saved searches remain available. diff --git a/docs/user_guide/client-usage.md b/docs/user_guide/client-usage.md deleted file mode 100644 index eccd43b..0000000 --- a/docs/user_guide/client-usage.md +++ /dev/null @@ -1,181 +0,0 @@ ---- -title: "Client Usage" -description: "Use the SerpApi client, module helpers, request options, and response helpers." ---- - -# Client Usage - -Create a `serpapi.Client` to reuse your API key, timeout, and HTTP connection settings across requests. - -## Create a Client - -First, [install the package and set `SERPAPI_KEY`](getting-started.md#installation). The examples on this page use the following client: - -```python -import os -import serpapi - -client = serpapi.Client( - api_key=os.environ["SERPAPI_KEY"], - timeout=20, -) -``` - -`timeout=20` sets the default connection and read timeout in seconds for this client's requests. It applies to searches, archive lookups, account and location requests, and image uploads. See [Errors and Timeouts](errors-and-timeouts.md#connection-errors-and-timeouts) for how the timeout works. - -## Search - -Pass search parameters by name, using Python keyword arguments: - -```python -results = client.search(engine="google", q="coffee") -``` - -You can also pass a dictionary when parameters are already stored in one: - -```python -params = { - "engine": "google", - "q": "coffee", -} -results = client.search(params) -``` - -You can combine a dictionary with keyword arguments. The client adds the keyword arguments to the dictionary and replaces existing values with the same name: - -```python -params = {"engine": "google", "q": "coffee"} -results = client.search(params, location="Austin, Texas") -``` - -## Module Helpers - -You can call functions such as `serpapi.search()` directly without creating a client: - -```python -import serpapi - -results = serpapi.search( - api_key="secret_api_key", - engine="google", - q="coffee", -) -``` - -These functions use a shared client. Create your own `serpapi.Client` when you need separate settings for an application, test, or worker that runs searches. - -## Search Archive - -Use `search_archive()` to retrieve a saved search. Get its ID from the original response: - -```python -search_id = results["search_metadata"]["id"] -archived = client.search_archive(search_id=search_id) -``` - -If `search_id` is missing, the client raises `serpapi.SearchIDNotProvided`. - -## Account and Locations - -```python -account = client.account() -locations = client.locations(q="Austin", limit=3) -``` - -`account()` returns account information for your API key. `locations()` returns supported Google locations that match the query. See [Account and Locations](account-and-locations.md) for examples and how to use a location in a search. - -## Request Options - -`search()`, `search_archive()`, `account()`, and `locations()` pass these keyword arguments to the underlying `requests` call: - -| Option | Use | -| --- | --- | -| `timeout` | Set a different timeout, in seconds, for one request. | -| `proxies` | Send the request through a proxy. | -| `verify` | Check the server's TLS certificate, or use a custom certificate authority bundle. | -| `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | -| `cert` | Authenticate the request with a client certificate. | - -For example, set a shorter timeout for this search: - -```python -results = client.search( - engine="google", - q="coffee", - timeout=10, -) -``` - -See [Request Options](request-options.md) for proxy, TLS, certificate, and per-request timeout examples. - -## Response Objects - -For JSON searches, the client returns a `serpapi.SerpResults` object. Read its fields like a dictionary: - -```python -first = results["organic_results"][0] -print(first.get("title")) -print(first.get("link")) -``` - -Use `as_dict()` when another library needs a plain dictionary: - -```python -payload = results.as_dict() -``` - -Use `output="html"` to receive the original search results page as an HTML string: - -```python -html = client.search(engine="google", q="coffee", output="html") -print(html[:500]) -``` - -Use `output="md"` to receive a Markdown string, for example to pass search results to an AI agent: - -```python -markdown = client.search(engine="google", q="coffee", output="md") -print(markdown[:500]) -``` - -See [Output Formats](output-formats.md) for details on JSON, Markdown, and HTML responses. - -## Pagination Helpers - -For one additional page, call `next_page()`: - -```python -next_results = results.next_page() -``` - -Use `yield_pages()` in a loop to read the current page and fetch more pages. This example reads up to ten pages and stops if a page contains an `error` message: - -```python -for page_number, page in enumerate(results.yield_pages(max_pages=10), start=1): - if "error" in page: - print(page["error"]) - break - current = page.get("serpapi_pagination", {}).get("current", page_number) - print(current) -``` - -See [Pagination](pagination.md) for more examples and how to use Google's `start` parameter. - -## Error Handling - -Use `try` and `except` to handle a timeout, a connection failure, or an HTTP error: - -```python -import serpapi - -try: - results = client.search(engine="google", q="coffee") -except serpapi.TimeoutError: - print("The request timed out.") -except serpapi.HTTPConnectionError: - print("Could not connect to SerpApi.") -except serpapi.HTTPError as exc: - print(exc.status_code, exc.error) -``` - -See [Errors and Timeouts](errors-and-timeouts.md) for error details and timeout settings. diff --git a/docs/user_guide/errors-and-timeouts.md b/docs/user_guide/errors-and-timeouts.md deleted file mode 100644 index 58a2814..0000000 --- a/docs/user_guide/errors-and-timeouts.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -title: "Errors and Timeouts" -description: "Handle SerpApi HTTP errors, connection errors, timeouts, and API-level errors." ---- - -# Errors and Timeouts - -The client raises exceptions when a request times out, cannot connect, or receives an HTTP error. Use `try` and `except` to decide what your script should do when a request fails. - -The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). - -## HTTP Errors - -HTTP responses with a 4xx or 5xx status code raise `serpapi.HTTPError`: - -```python -import serpapi - -try: - results = client.search(engine="google", q="coffee") -except serpapi.HTTPError as exc: - print("Status:", exc.status_code) - print("Error:", exc.error) -``` - -`exc.status_code` contains the HTTP status code, such as `401` for an invalid API key. `exc.error` contains the error message from SerpApi's JSON response, or `None` if the response has no JSON error message. - -See [SerpApi API status and error codes](https://serpapi.com/api-status-and-error-codes) for the meaning of each status and how to resolve it. - -## Connection Errors and Timeouts - -Catch `TimeoutError` when a request takes too long to connect or receive data. Catch `HTTPConnectionError` when the client cannot establish a connection: - -```python -try: - results = client.search(engine="google", q="coffee", timeout=10) -except serpapi.TimeoutError: - print("The request timed out.") -except serpapi.HTTPConnectionError: - print("Could not connect to SerpApi.") -``` - -Timeouts are measured in seconds. Set a default for all requests made by a client: - -```python -client = serpapi.Client(api_key="secret_api_key", timeout=20) -``` - -Override it for one request: - -```python -results = client.search( - engine="google", - q="coffee", - timeout=5, -) -``` - -The timeout applies to connecting and waiting for data. A request can take longer than the timeout overall if data continues to arrive. Without a timeout setting, the client can wait indefinitely. See the [Requests timeout documentation](https://requests.readthedocs.io/en/latest/user/quickstart/#timeouts). - -## Missing Search IDs - -`search_archive()` requires the ID of a previous search. If you leave out `search_id`, it raises `SearchIDNotProvided`: - -```python -try: - archived = client.search_archive() -except serpapi.SearchIDNotProvided: - print("Provide search_id from search_metadata.id.") -``` - -## API-Level Errors in JSON - -A response can contain an `error` message even when the HTTP request succeeds. Check for this field before reading the results: - -```python -results = client.search(engine="google", q="coffee") - -if "error" in results: - raise RuntimeError(results["error"]) -``` diff --git a/docs/user_guide/extract.md b/docs/user_guide/extract.md deleted file mode 100644 index 0fa6650..0000000 --- a/docs/user_guide/extract.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: "Extract API" -description: "Extract page content, discover URLs, and crawl websites with the Python client." ---- - -# Extract API - -Use `client.extract()` to read content from a public URL. It supports single-page scraping, URL discovery, and crawling. Pass a parameter dictionary or keyword arguments, as you would with `client.search()`. The client sets `engine="extract"` automatically. - -See the [Extract API documentation](https://serpapi.com/extract-api) for all parameters and response fields. - -## Get Page Content as Markdown - -Set `output="md"` to get the page's main content as a Markdown string. - -After [setting your API key](getting-started.md#installation), run: - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=60) -markdown = client.extract( - url="https://serpapi.com/", - output="md", -) - -print(markdown) -``` - -You can pass the returned string to an AI agent or save it to a file. - -## Get Structured Data as JSON - -JSON is the default output format. The SDK returns a dictionary-like `SerpResults` object with page information, meta tags, structured data, links, images, and Markdown. The sections returned depend on the target page. - -```python -results = client.extract(url="https://serpapi.com/") - -print(results["page_information"]["title"]) -print(results["markdown"]) -``` - -The `markdown` field contains the page's main content. Set `include_markdown=False` to omit it from the JSON response. - -The module-level `serpapi.extract(url=..., api_key=...)` function accepts the same parameters. - -## Get HTML - -Set `output="html"` to get the retrieved HTML as a string: - -```python -html = client.extract(url="https://serpapi.com/", output="html") -``` - -## Discover URLs - -Set `mode="map"` to discover URLs on a website: - -```python -results = client.extract( - url="https://serpapi.com/", - mode="map", - include_sitemap=False, - include_subdomains=False, -) - -for entry in results["urls"]: - print(entry["url"]) -``` - -Map mode includes sitemap URLs by default. Set `include_sitemap=False` to discover URLs from the page's links only. Set `include_subdomains=True` to include subdomains. - -## Crawl Multiple Pages - -Set `mode="crawl"` to follow links and extract pages: - -```python -results = client.extract( - url="https://serpapi.com/", - mode="crawl", - max_pages=2, - max_depth=1, - timeout=180, -) - -for page in results["pages"]: - print(page["url"]) - print(page["markdown"]) -``` - -Each entry in `results["pages"]` contains the extracted page's data, including its `markdown` field. The starting page has depth `0`. `max_pages` includes the starting page and defaults to `10`. Each successfully extracted page can consume one search credit. Set a page limit for your crawl and allow enough time for the HTTP request to finish. - -For requests you want to retrieve later, pass `"async": True` in a parameter dictionary and use the [Search Archive](async-search-archive.md). - -## Timeouts and Content Filters - -The `timeout=` keyword sets the SDK's HTTP timeout in seconds. Set it above the API fetch timeout, and allow time for all pages when crawling. Put the Extract API's fetch timeout in the parameter dictionary to send it in milliseconds: - -```python -results = client.extract( - {"url": "https://serpapi.com/", "timeout": 30000}, - timeout=60, -) -``` - -Use `include_tags` and `exclude_tags` to keep or remove content with CSS selectors, such as `include_tags="article, main"` or `exclude_tags="nav, footer"`. See the [Extract API parameters](https://serpapi.com/extract-api) for details and [Errors and Timeouts](errors-and-timeouts.md) for exception handling. diff --git a/docs/user_guide/getting-started.md b/docs/user_guide/getting-started.md deleted file mode 100644 index 99490b1..0000000 --- a/docs/user_guide/getting-started.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -title: "Getting Started" -description: "Install serpapi, create a client, and run your first search." ---- - -# Getting Started - -The `serpapi` package lets you request search results from [SerpApi](https://serpapi.com) in Python. You need Python installed and a SerpApi account with an API key. - -## Installation - -Run one of these commands in your terminal. Use `pip` for an existing Python environment, `uv add` for a uv project, or `uv pip install` for a virtual environment managed with uv. - -::::{tab-set} - -:::{tab-item} pip - -```bash -pip install serpapi -``` - -::: - -:::{tab-item} uv project - -```bash -uv add serpapi -``` - -::: - -:::{tab-item} uv environment - -```bash -uv pip install serpapi -``` - -::: - -:::: - -The package requires Python 3.6 or newer. - -Create or sign in to your SerpApi account and copy your API key from the [dashboard](https://serpapi.com/manage-api-key). An API key identifies your account when you make a request. Replace `secret_api_key` below with your key and run the command in your terminal. On Windows, use PowerShell. - -::::{tab-set} - -:::{tab-item} macOS / Linux - -```bash -export SERPAPI_KEY="secret_api_key" -``` - -::: - -:::{tab-item} Windows - -```powershell -$env:SERPAPI_KEY = "secret_api_key" -``` - -::: - -:::: - -## First Search - -Save this code in a file named `search_example.py`. Run it with `python search_example.py` from the same terminal where you set the key. Use `python3` if that is the command for Python on your computer. If you installed the package with `uv add`, run `uv run python search_example.py` from your project directory. - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - -results = client.search( - engine="google", - q="coffee shops", - location="Austin, Texas", - hl="en", - gl="us", -) - -first_result = results["organic_results"][0] -print(first_result["title"]) -print(first_result["link"]) -``` - -`os.environ["SERPAPI_KEY"]` reads the key you set in the terminal. If Python raises `KeyError: 'SERPAPI_KEY'`, set the variable in the terminal where you run the script, or in your editor's run configuration. - -`client.search()` sends the search to SerpApi. By default, the response uses JSON, a format for named fields and lists. The client converts it into a `SerpResults` object, which you can read like a Python dictionary. It also has methods such as `as_dict()`, `next_page()`, and `yield_pages()`. - -`results["organic_results"]` is a list of unpaid search results. Python counts list positions from zero, so `[0]` selects the first result. The example prints its title and link. For searches that may return no results, use the loop shown in [Output Formats](output-formats.md#json). - -## What the Parameters Mean - -The example above sends a Google Search request: - -| Parameter | Purpose | -| --- | --- | -| `engine` | Selects the SerpApi engine. `google` is Google Search. | -| `q` | Text to search for, such as `coffee shops`. | -| `location` | Location to search from, such as `Austin, Texas`. | -| `hl` | Language for the Google interface. `en` is English. | -| `gl` | Country for Google results. `us` is the United States. | - -Each engine has its own search parameters, listed in the [SerpApi API documentation](https://serpapi.com/search-api). Use the [SerpApi Playground](https://serpapi.com/playground) to try a search in your browser and copy the parameters into Python. - -## When to Use the Client - -Create a `serpapi.Client` when you need to make several requests, fetch more pages, or run searches at the same time. You can reuse its API key, timeout, and HTTP session, which manages connections to SerpApi. - -For a single search, you can also call `serpapi.search()` directly: - -```python -import serpapi - -results = serpapi.search( - api_key="secret_api_key", - engine="google", - q="coffee", -) -``` - -To reuse the same settings for later searches, create a client: - -```python -client = serpapi.Client(api_key="secret_api_key", timeout=20) -results = client.search(engine="google", q="coffee") -``` - -## Next Steps - -- Read [Client Usage](client-usage.md) for response helpers, request options, and archive helpers. -- Read [Pagination](pagination.md) when collecting more than one page of results. -- Try [Google Across Countries](../examples/google-across-countries.md) to compare results from different countries. diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md deleted file mode 100644 index 4e58543..0000000 --- a/docs/user_guide/index.md +++ /dev/null @@ -1,30 +0,0 @@ -# User Guide - -Start with [Getting Started](getting-started.md) to install the package and run a search. The other guides explain how to read results, fetch more pages, and handle errors. - -## Basics - -- [Getting Started](getting-started.md) -- [Client Usage](client-usage.md) - -## User Guide - -- [Parameters and Engines](parameters-and-engines.md) -- [Output Formats](output-formats.md) -- [Extract API](extract.md) -- [Pagination](pagination.md) -- [Errors and Timeouts](errors-and-timeouts.md) -- [Account and Locations](account-and-locations.md) - -## Migration - -- [Migrating from google-search-results](migrating-from-google-search-results.md) - -## Advanced Usage - -- [Async Search Archive](async-search-archive.md) -- [Threading](threading.md) -- [Multiprocessing](multiprocessing.md) -- [Zero Trace and Data Retention](zero-trace.md) -- [JSON Restrictor](json-restrictor.md) -- [Request Options](request-options.md) diff --git a/docs/user_guide/json-restrictor.md b/docs/user_guide/json-restrictor.md deleted file mode 100644 index 99506df..0000000 --- a/docs/user_guide/json-restrictor.md +++ /dev/null @@ -1,137 +0,0 @@ ---- -title: "JSON Restrictor" -description: "Request selected JSON fields with json_restrictor." ---- - -# JSON Restrictor - -Use `json_restrictor` to ask SerpApi to return selected fields. For example, you can request titles and links while leaving out images and other sections you do not need. SerpApi filters the response before sending it to your program. - -The parameter works across SerpApi engines. Its value is a selector string that names the fields to keep. See the [SerpApi JSON Restrictor documentation](https://serpapi.com/json-restrictor) for all selector operators. - -Create a client as shown in [Client Usage](client-usage.md#create-a-client) before running the examples. - -## Select a Top-Level Section - -Return only `organic_results`: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - json_restrictor="organic_results", -) - -for result in results.get("organic_results", []): - print(result.get("title")) -``` - -## Select Fields From Each Result - -Use `[]` to select every item in a JSON array, which becomes a list in Python. List field names inside `.{...}` to keep several fields from each item: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - json_restrictor="organic_results[].{title, link, snippet}", -) - -for result in results.get("organic_results", []): - print(result["title"], result["link"]) -``` - -The response keeps the `organic_results` list. Each result in that list contains only the selected fields that are present. - -## Select One Item or a Slice - -Use `[0]` to select the first item in a list. As in Python, indexes start at zero: - -```python -results = client.search( - engine="google", - q="coffee", - json_restrictor="organic_results[0]", -) - -first_result = results["organic_results"][0] -print(first_result["title"]) -``` - -Use a slice to select a range of items. The start index is included and the end index is excluded: - -```python -results = client.search( - engine="google", - q="coffee", - json_restrictor="organic_results[0:3]", -) -``` - -`organic_results[0:3]` keeps indexes `0`, `1`, and `2`, so it returns at most three items. - -## Combine Multiple Selectors - -Separate selectors with commas to keep more than one part of the response: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - json_restrictor="local_map, organic_results[0]", -) - -print(results.get("local_map", {}).get("gps_coordinates")) -print(results["organic_results"][0]["title"]) -``` - -## Nested Fields - -Use dots to select fields inside other fields. This selector keeps each result's title and the links in its `sitelinks.inline` list: - -```python -results = client.search( - engine="google", - q="coffee", - json_restrictor="organic_results[].{title, sitelinks.inline[].link}", -) -``` - -The selected links remain inside `sitelinks.inline`, so your code can read them at the same location as in the full response. - -## Common Selector Patterns - -| Need | `json_restrictor` | -| --- | --- | -| One top-level section | `organic_results` | -| First organic result | `organic_results[0]` | -| First three organic results | `organic_results[0:3]` | -| One field from every result | `organic_results[].title` | -| Multiple fields from every result | `organic_results[].{title, snippet}` | -| Nested fields | `organic_results[].{title, sitelinks.inline[].link}` | -| Multiple response sections | `local_map, organic_results[0]` | - -## Practical Guidance - -- First make a search without `json_restrictor` and inspect the response. Then add selectors for the fields you need. -- Check selectors against a response from the engine you are using. Result sections and field names vary between engines. -- If you plan to call `next_page()` or `yield_pages()`, include `serpapi_pagination` in the selector so the client can find the next-page URL. -- If you need a search ID, status, or archive metadata, include `search_metadata`. - -To fetch more pages, keep `serpapi_pagination` alongside the result fields: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - json_restrictor="organic_results[].{title, link}, serpapi_pagination", -) - -for page in results.yield_pages(max_pages=3): - for result in page.get("organic_results", []): - print(result["title"], result["link"]) -``` diff --git a/docs/user_guide/migrating-from-google-search-results.md b/docs/user_guide/migrating-from-google-search-results.md deleted file mode 100644 index e074a1b..0000000 --- a/docs/user_guide/migrating-from-google-search-results.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: "Migrating from google-search-results" -description: "Move from the deprecated google-search-results SDK to the recommended serpapi package." ---- - -# Migrating from google-search-results - -Both `google-search-results` and `serpapi` use the name `serpapi` in Python imports. Check which package your project installs before updating the code. - -## Recommended Package - -Use [`serpapi` on PyPI](https://pypi.org/project/serpapi/) for new projects and when updating an existing integration. - -Install it with: - -```bash -pip install serpapi -``` - -After [setting your API key](getting-started.md#installation), create a `serpapi.Client`: - -```python -import os -import serpapi - -YOUR_API_KEY = os.environ["SERPAPI_KEY"] - -client = serpapi.Client(api_key=YOUR_API_KEY) -results = client.search(engine="google", q="coffee") - -print(results) -``` - -This is the package used throughout the current documentation. - -## Deprecated Package - -[`google-search-results` on PyPI](https://pypi.org/project/google-search-results/) is the older Python package. It is deprecated for new integrations. - -It was installed with: - -```bash -pip install google-search-results -``` - -Older code often looks like this: - - -```python -from serpapi import GoogleSearch - -search = GoogleSearch({ - "q": "coffee", - "location": "Austin,Texas", - "api_key": "", -}) -result = search.get_dict() -``` - -Do not add this package to new projects. - -## Update Your Dependencies - -Remove `google-search-results` from your dependency files, such as `requirements.txt`, `pyproject.toml`, `setup.py`, `Pipfile`, or Poetry dependency configuration. - -Then add `serpapi` instead: - -```bash -pip install serpapi -``` - -Install only one of these packages in a Python environment. Both provide a `serpapi` module, so installing them together can overwrite files and cause import errors. - -## Update Your Code - -Replace `GoogleSearch(...).get_dict()` with `serpapi.Client(...).search(...)`. - -Old: - - -```python -from serpapi import GoogleSearch - -search = GoogleSearch({ - "q": "coffee", - "location": "Austin,Texas", - "api_key": "", -}) -result = search.get_dict() -``` - -New: - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", -) -``` - -Search parameter names stay the same. You can pass them by name, as in the new example, or keep them in a dictionary and pass that to `client.search()`. - -## Migration Checklist - -- Remove `google-search-results` from requirements and dependency files. -- Install `serpapi` with `pip install serpapi`. -- Replace `from serpapi import GoogleSearch` with `import serpapi`. -- Replace `GoogleSearch(params).get_dict()` with `serpapi.Client(api_key=...).search(engine=..., q=..., ...)`. -- Keep using the [SerpApi Playground](https://serpapi.com/playground) and the [SerpApi API documentation](https://serpapi.com/search-api) to confirm engine parameters. diff --git a/docs/user_guide/multiprocessing.md b/docs/user_guide/multiprocessing.md deleted file mode 100644 index fd4e041..0000000 --- a/docs/user_guide/multiprocessing.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: "Multiprocessing" -description: "Run searches in separate Python processes with ProcessPoolExecutor." ---- - -# Multiprocessing - -Multiprocessing runs work in separate Python processes. Use it when you do substantial computation on each search response, need separate process memory, or already use a process pool in your application. [Threads](threading.md) have less startup overhead for searches that mainly wait for network responses. - -## ProcessPoolExecutor Example - -After [setting your API key](getting-started.md#installation), save this example in a Python file and run it from your terminal. `max_workers=4` allows up to four worker processes. Each worker creates its own client and returns a dictionary with selected results. - -```python -import os -import serpapi - -from concurrent.futures import ProcessPoolExecutor, as_completed - - -def search_country(country): - client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - results = client.search( - engine="google", - q="best coffee beans", - gl=country, - hl="en", - ) - - organic_results = results.get("organic_results", []) - return { - "country": country, - "count": len(organic_results), - "first_title": organic_results[0].get("title") if organic_results else None, - } - - -if __name__ == "__main__": - countries = ["us", "gb", "ca", "au", "in"] - - with ProcessPoolExecutor(max_workers=4) as executor: - futures = [executor.submit(search_country, country) for country in countries] - - for future in as_completed(futures): - print(future.result()) -``` - -Keep `if __name__ == "__main__":` around the code that starts the pool. When a new worker imports the file, this guard prevents it from starting another pool. It is required when workers start this way, as they do by default on macOS and Windows. - -## Practical Guidance - -- Create the `serpapi.Client` inside the worker process. -- Return data that Python can copy between processes, such as dictionaries, lists, strings, and numbers. -- Avoid passing open sessions, response objects, or client instances between processes. -- Set a timeout on each client so a stalled request does not wait indefinitely. -- Use threads if your workers only make searches and read a few fields from the response. - -## Combining Pagination and Processes - -You can submit a different Google `start` offset to each worker. Here is a worker function that fetches one range of results: - -```python -def search_offset(start): - client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) - results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - start=start, - ) - return results.get("organic_results", []) -``` - -Use the [pagination parameters](pagination.md#pagination-parameters-vary-by-engine) for the engine you are searching. Other engines may require a token from the previous page to request the next one. diff --git a/docs/user_guide/output-formats.md b/docs/user_guide/output-formats.md deleted file mode 100644 index d779b0a..0000000 --- a/docs/user_guide/output-formats.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: "Output Formats" -description: "Choose JSON, Markdown, or HTML results with the output parameter." ---- - -# Output Formats - -Set `output` on `client.search()` to choose the response format. JSON is the default. - -| `output` | Python return type | Use | -| --- | --- | --- | -| `"json"` | `serpapi.SerpResults` | Read individual fields, such as titles and links, and fetch more pages. | -| `"md"` | `str` | Give Markdown search results to AI agents and language models. | -| `"html"` | `str` | Inspect the search results as HTML. | - -## JSON - -JSON stores results as named fields and lists. Use it when your code needs to read individual values, such as a result's title or link. - -After [setting your API key](getting-started.md#installation), create a client and run a search: - -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) -results = client.search(engine="google", q="coffee", output="json") - -for result in results.get("organic_results", []): - print(result.get("title"), result.get("link")) -``` - -You can leave out `output="json"` because JSON is the default. `SerpResults` behaves like a dictionary and provides `as_dict()`, `next_page()`, and `yield_pages()`. The loop above uses an empty list if `organic_results` is missing. See [Pagination](pagination.md) for examples that fetch more pages. - -## Markdown - -Use `output="md"` for search results formatted with headings, links, and tables: - -```python -markdown = client.search(engine="google", q="coffee", output="md") -print(markdown[:500]) -``` - -The client returns a plain Python string. The example prints its first 500 characters. You can pass the full string to an AI agent as the result of a search tool. See [AI Agents](../ai-agents.md) for integrations and [SerpApi's Markdown output guide](https://serpapi.com/markdown-output) for API details. - -## HTML - -Use `output="html"` to retrieve the HTML of the original search results page: - -```python -html = client.search(engine="google", q="coffee", output="html") -print(html[:500]) -``` - -The client returns HTML as a string. Use JSON when you need to read named result fields or call pagination methods such as `next_page()`. These methods are available on `SerpResults`, and cannot be called on an HTML or Markdown string. - -The same `output` values are supported by `client.search_archive()` when retrieving a saved search. See [Async Search Archive](async-search-archive.md). diff --git a/docs/user_guide/pagination.md b/docs/user_guide/pagination.md deleted file mode 100644 index 7999685..0000000 --- a/docs/user_guide/pagination.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: "Pagination" -description: "Collect additional pages with SerpResults helpers or engine-specific offsets." ---- - -# Pagination - -Pagination means fetching more than one page of search results. If a JSON response includes a next-page link in `serpapi_pagination`, you can read it with `results.next_page_url` or fetch pages with `next_page()` and `yield_pages()`. - -The examples below use a client created as shown in [Client Usage](client-usage.md#create-a-client). - -## Fetch One More Page - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", -) - -next_results = results.next_page() - -if next_results: - for item in next_results.get("organic_results", []): - print(item.get("title")) -``` - -`next_page()` returns `None` when the response does not include a next page URL. - -## Iterate Through Pages - -Use `yield_pages()` when you want the current page plus following pages: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", -) - -for page in results.yield_pages(max_pages=5): - for item in page.get("organic_results", []): - print(item.get("position"), item.get("title")) -``` - -Set `max_pages` to limit how many pages your script reads, including the first page. Each additional page requires another request. The loop stops early if there is no next-page link, and results may change between requests. - -## Google `start` Offsets - -Google's `start` parameter tells SerpApi how many results to skip. For example, `start=10` skips the first ten results: - -```python -for start in [0, 10, 20]: - page = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - start=start, - ) - - print("Offset:", start) - for item in page.get("organic_results", []): - print(item.get("title")) -``` - -Set offsets yourself when you need particular result ranges or want to request several ranges at the same time. - -## Store Only What You Need - -To keep less data in memory or storage, copy the fields you need into a list of dictionaries: - -```python -records = [] - -for page in results.yield_pages(max_pages=3): - for item in page.get("organic_results", []): - records.append({ - "title": item.get("title"), - "link": item.get("link"), - "snippet": item.get("snippet"), - }) -``` - -## Pagination Parameters Vary by Engine - -`start` is common for Google Search, but other engines may use different pagination parameters. Check the relevant engine page in the [SerpApi Search API documentation](https://serpapi.com/search-api) or build the request in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/user_guide/parameters-and-engines.md b/docs/user_guide/parameters-and-engines.md deleted file mode 100644 index ffdf39f..0000000 --- a/docs/user_guide/parameters-and-engines.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: "Parameters and Engines" -description: "Choose a search engine and pass its parameters to the Python client." ---- - -# Parameters and Engines - -The `engine` parameter selects which search service to use. The Python client sends your parameters to SerpApi without checking them against a fixed list. You can use a new engine or parameter as soon as SerpApi supports it. - -## Full Engine and Parameter Reference - -Find the parameters for your search in these resources: - -- [SerpApi Search API documentation](https://serpapi.com/search-api) lists supported engines and engine-specific parameters. -- [SerpApi Playground](https://serpapi.com/playground) lets you test a search and copy the exact parameters into Python. -- [Account and Locations](account-and-locations.md) shows how to find valid `location` values from Python. - -The Search API docs include engines such as Google Search, Google Maps, Google Images, Google Shopping, Google Scholar, Google Jobs, Bing, DuckDuckGo, Yahoo, Yandex, Baidu, YouTube, eBay, Walmart, Naver, Apple App Store, Home Depot, and many more. Check the docs for the current full list. - -## Passing Parameters - -Create a client as shown in [Client Usage](client-usage.md#create-a-client). Pass parameters by name, using Python keyword arguments: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - hl="en", - gl="us", -) -``` - -You can also pass a dictionary when you already have parameters in one: - -```python -params = { - "engine": "google", - "q": "coffee", - "location": "Austin, Texas", - "hl": "en", - "gl": "us", -} -results = client.search(params) -``` - -## Common Google Parameters - -These are common Google Search parameters. Other engines have their own parameter names. - -| Parameter | Purpose | -| --- | --- | -| `engine` | Search engine. Use `google` for Google Search. | -| `q` | Text to search for. | -| `location` | Geographic location for the search. | -| `google_domain` | Google domain, such as `google.com` or `google.co.in`. | -| `gl` | Country code for the search. | -| `hl` | Interface language. | -| `start` | Number of results to skip when fetching another page. | -| `device` | Device type, such as desktop or mobile. | -| `output` | Response format: `json` (default), `html`, or `md`. | -| `async` | Submit a search now and retrieve its results later. | -| `no_cache` | Force SerpApi to fetch fresh results. | -| `zero_trace` | Skip storing the search on SerpApi's servers when your account supports ZeroTrace. | -| `json_restrictor` | Return only selected JSON fields from the API response. | - -For the full supported list and the exact meaning of each parameter, use the [SerpApi Search API documentation](https://serpapi.com/search-api). - -See [Output Formats](output-formats.md) for examples of each response format and its Python return type. - -## Engine-Specific Names - -Different engines use different query parameter names. For example: - -```python -google = client.search(engine="google", q="coffee") -ebay = client.search(engine="ebay", _nkw="coffee grinder") -youtube = client.search(engine="youtube", search_query="coffee brewing") -walmart = client.search(engine="walmart", query="coffee") -``` - -Select the engine in the [SerpApi Playground](https://serpapi.com/playground), try your search, and copy the generated parameters. diff --git a/docs/user_guide/request-options.md b/docs/user_guide/request-options.md deleted file mode 100644 index 60c70db..0000000 --- a/docs/user_guide/request-options.md +++ /dev/null @@ -1,137 +0,0 @@ ---- -title: "Request Options" -description: "Pass requests library options such as timeout, proxies, verify, stream, and cert." ---- - -# Request Options - -Search parameters such as `engine`, `q`, `location`, `hl`, `gl`, `no_cache`, and `json_restrictor` tell SerpApi what to search for and return. Request options such as `timeout` and `proxies` control how your Python program connects to SerpApi through the `requests` library. - -The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). - -## Supported Request Options - -The client passes these keyword arguments to `requests.Session.request()`: - -| Option | Use | -| --- | --- | -| `timeout` | Set a different timeout, in seconds, for one request. | -| `proxies` | Send the request through HTTP or HTTPS proxies. | -| `verify` | Check the server's TLS certificate, or use a custom certificate authority bundle. | -| `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | -| `cert` | Authenticate the request with a client certificate. | - -These options are supported by `search()`, `search_archive()`, `account()`, and `locations()`. - -## Per-Request Timeout - -Set a default timeout when creating the client: - -```python -client = serpapi.Client(api_key="secret_api_key", timeout=20) -``` - -Override it for a single request: - -```python -results = client.search( - engine="google", - q="coffee", - timeout=5, -) -``` - -`timeout=5` sets the connection and read timeout to five seconds. It is passed to `requests`. See [Errors and Timeouts](errors-and-timeouts.md#connection-errors-and-timeouts) for timeout behavior. - -## Proxies - -To connect through a proxy server, pass a dictionary that maps the request protocol to the proxy URL: - - -```python -results = client.search( - engine="google", - q="coffee", - proxies={ - "https": "http://proxy.example.com:8080", - }, -) -``` - -Keep proxy credentials in environment variables or your secret manager rather than hardcoding them in source files. - -## TLS Verification - -By default, `requests` checks the server's TLS certificate to verify its identity. If your network uses a private certificate authority (CA), pass the path to its trusted certificate bundle: - - -```python -results = client.search( - engine="google", - q="coffee", - verify="/path/to/ca-bundle.pem", -) -``` - -Only disable verification for controlled local debugging: - - -```python -results = client.search( - engine="google", - q="coffee", - verify=False, -) -``` - -Do not use `verify=False` in production code. - -## Client Certificates - -Some servers or proxies require a certificate to identify the client. Pass its file path with `cert`, or pass a `(cert, key)` tuple if the certificate and private key are in separate files: - - -```python -results = client.search( - engine="google", - q="coffee", - cert=("/path/to/client-cert.pem", "/path/to/client-key.pem"), -) -``` - -## Combining Search Parameters and Request Options - -Request options can be used alongside normal SerpApi parameters: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - hl="en", - gl="us", - json_restrictor="organic_results[].{title, link}", - timeout=10, -) -``` - -The client sends the arguments to two places: - -- `timeout`, `proxies`, `verify`, `stream`, and `cert` are passed to `requests`. -- All remaining keyword arguments are sent to SerpApi as API parameters. - -If you pass a parameter dictionary and keyword arguments together, the search parameters in the keyword arguments update your dictionary: - -```python -params = {"engine": "google", "q": "coffee"} -results = client.search(params, location="Austin, Texas", timeout=10) -``` - -In this example, `location` is added to the SerpApi request parameters, while `timeout` is passed to `requests`. - -When you want to keep the original dictionary unchanged, pass a copy: - -```python -params = {"engine": "google", "q": "coffee"} -results = client.search(params.copy(), location="Austin, Texas") -``` diff --git a/docs/user_guide/threading.md b/docs/user_guide/threading.md deleted file mode 100644 index 7b7259b..0000000 --- a/docs/user_guide/threading.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: "Threading" -description: "Run several independent searches at the same time with ThreadPoolExecutor." ---- - -# Threading - -Search requests spend time waiting for network responses. Threads let your program make another request while one is waiting. Use them for independent searches, such as running the same query in several countries. - -## ThreadPoolExecutor Example - -After [setting your API key](getting-started.md#installation), run this example. `max_workers=5` allows up to five searches to run at once. Each submitted task returns a `Future`, an object that holds its eventual result or exception. `as_completed()` gives you each task as it finishes. - -```python -import os -import serpapi - -from concurrent.futures import ThreadPoolExecutor, as_completed - - -API_KEY = os.environ["SERPAPI_KEY"] - - -def search_country(country): - client = serpapi.Client(api_key=API_KEY, timeout=20) - results = client.search( - engine="google", - q="best coffee beans", - gl=country, - hl="en", - ) - return country, results.get("organic_results", []) - - -countries = ["us", "gb", "ca", "au", "in"] - -with ThreadPoolExecutor(max_workers=5) as executor: - futures = [executor.submit(search_country, country) for country in countries] - - for future in as_completed(futures): - country, organic_results = future.result() - first = organic_results[0] if organic_results else {} - print(country, first.get("title")) -``` - -## Practical Guidance - -- Start with a small `max_workers` value. Increase it based on your account's search limits, response times, and how quickly your code can process results. -- Handle each task's exception so the loop can continue processing other results. -- Create a client inside each worker to give it a separate HTTP session. -- Set a timeout so a stalled request does not hold up the batch indefinitely. - -## Handling Per-Request Errors - -`future.result()` raises any exception from the search. Catch it inside the loop to report the failed country and continue: - -```python -with ThreadPoolExecutor(max_workers=5) as executor: - future_to_country = { - executor.submit(search_country, country): country - for country in countries - } - - for future in as_completed(future_to_country): - country = future_to_country[future] - - try: - country, organic_results = future.result() - except serpapi.SerpApiError as exc: - print("Failed:", country, exc) - continue - - print("Finished:", country, len(organic_results)) -``` diff --git a/docs/user_guide/zero-trace.md b/docs/user_guide/zero-trace.md deleted file mode 100644 index c6fcfef..0000000 --- a/docs/user_guide/zero-trace.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: "Zero Trace and Data Retention" -description: "Control search storage and request fresh results with zero_trace and no_cache." ---- - -# Zero Trace and Data Retention - -Use `zero_trace` to control whether SerpApi stores a search, and `no_cache` to request fresh results. Pass these parameters to a client created as shown in [Client Usage](client-usage.md#create-a-client). - -## Zero Trace - -If your account supports ZeroTrace, pass `zero_trace=True` to skip storing search parameters, files, and metadata on SerpApi's servers: - -```python -results = client.search( - engine="google", - q="coffee", - location="Austin, Texas", - zero_trace=True, -) -``` - -Check your plan and the [SerpApi Search API documentation](https://serpapi.com/search-api) for ZeroTrace availability and data retention details. - -## Cache Controls - -SerpApi can reuse a saved response when the search parameters match a recent request. Pass `no_cache=True` to fetch fresh results: - -```python -results = client.search( - engine="google", - q="coffee", - no_cache=True, -) -``` - -`no_cache=true` and `async=true` should not be used together. - -## Choosing Between Modes - -| Need | Parameter | -| --- | --- | -| Fetch fresh results | `no_cache=true` | -| Submit now and fetch later | `async=true` plus `search_archive()` | -| Skip storing the search on SerpApi's servers | `zero_trace=true` when enabled for your account | - -The table uses API parameter syntax. In Python, write `True` with a capital `T`. See [Async Search Archive](async-search-archive.md) for how to pass the `async` parameter. diff --git a/examples/async_batch_search.py b/examples/async_batch_search.py new file mode 100644 index 0000000..65121f6 --- /dev/null +++ b/examples/async_batch_search.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Async batch search example for SerpApi Python client. + +This example demonstrates how to perform multiple searches concurrently +using async/await for better performance. +""" + +import asyncio +import os + +from serpapi import Client, SerpApiError + + +async def search_company(client: Client, company: str) -> dict: + """Search for a specific company.""" + try: + results = await client.search({"q": company}) + return { + "company": company, + "status": results.get("search_metadata", {}).get("status", "Unknown"), + "search_id": results.get("search_metadata", {}).get("id", "N/A"), + "organic_count": len(results.get("organic_results", [])), + "success": True, + } + except SerpApiError as e: + return {"company": company, "error": str(e), "success": False} + + +async def main(): + """Main example function.""" + # Get API key from environment variable + api_key = os.getenv("SERPAPI_KEY") + if not api_key: + print("Please set SERPAPI_KEY environment variable") + return + + # Create client with persistent connections for better performance + client = Client(api_key=api_key, engine="google", persistent=True) + + # List of companies to search + companies = ["meta", "amazon", "apple", "netflix", "google", "microsoft", "tesla"] + + try: + print("=== Async Batch Search ===") + print(f"Searching for {len(companies)} companies concurrently...") + print() + + # Create tasks for concurrent execution + tasks = [search_company(client, company) for company in companies] + + # Execute all searches concurrently + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Process results + successful_searches = [] + failed_searches = [] + + for result in results: + if isinstance(result, Exception): + print(f"Unexpected error: {result}") + continue + + if result["success"]: + successful_searches.append(result) + print( + f"✓ {result['company']}: {result['organic_count']} results " + f"(Status: {result['status']}, ID: {result['search_id'][:8]}...)" + ) + else: + failed_searches.append(result) + print(f"✗ {result['company']}: {result['error']}") + + print() + print( + f"Summary: {len(successful_searches)} successful, {len(failed_searches)} failed" + ) + + # Demonstrate search archive functionality + if successful_searches: + print("\n=== Search Archive Example ===") + first_result = successful_searches[0] + search_id = first_result["search_id"] + + try: + archived_result = await client.search_archive(search_id) + print(f"Retrieved archived result for {first_result['company']}") + print( + f"Archive status: {archived_result.get('search_metadata', {}).get('status', 'Unknown')}" + ) + except SerpApiError as e: + print(f"Failed to retrieve archive: {e}") + + except Exception as e: + print(f"Unexpected error: {e}") + finally: + # Close the client + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/basic_usage.py b/examples/basic_usage.py new file mode 100644 index 0000000..3935132 --- /dev/null +++ b/examples/basic_usage.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Basic usage example for SerpApi Python client. + +This example demonstrates how to use the SerpApi client for basic search operations. +""" + +import asyncio +import os + +from serpapi import Client, SerpApiError + + +async def main(): + """Main example function.""" + # Get API key from environment variable + api_key = os.getenv("SERPAPI_KEY") + if not api_key: + print("Please set SERPAPI_KEY environment variable") + return + + # Create client + client = Client(api_key=api_key, engine="google") + + try: + # Basic search + print("=== Basic Google Search ===") + results = await client.search({"q": "coffee"}) + + if "organic_results" in results: + print(f"Found {len(results['organic_results'])} organic results:") + for i, result in enumerate(results["organic_results"][:3], 1): + print(f"{i}. {result.get('title', 'No title')}") + print(f" {result.get('link', 'No link')}") + print() + else: + print("No organic results found") + + # HTML search + print("=== HTML Search ===") + html_content = await client.html({"q": "python programming"}) + print(f"HTML content length: {len(html_content)} characters") + print(f"First 200 characters: {html_content[:200]}...") + print() + + # Location search + print("=== Location Search ===") + locations = await client.location({"q": "Austin", "limit": 3}) + print(f"Found {len(locations)} locations:") + for location in locations: + print( + f"- {location.get('canonical_name', 'No canonical name')} " + ) + print() + + # Account information + print("=== Account Information ===") + account = await client.account() + print(f"Account ID: {account.get('account_id', 'N/A')}") + print(f"Plan: {account.get('plan_name', 'N/A')}") + print(f"Searches left: {account.get('total_searches_left', 'N/A')}") + + except SerpApiError as e: + print(f"SerpApi Error: {e}") + except Exception as e: + print(f"Unexpected error: {e}") + finally: + # Close the client + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/multiple_engines.py b/examples/multiple_engines.py new file mode 100644 index 0000000..8d23bb9 --- /dev/null +++ b/examples/multiple_engines.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +Multiple search engines example for SerpApi Python client. + +This example demonstrates how to use different search engines +and compare their results. +""" + +import asyncio +import os + +from serpapi import Client, SerpApiError + + +async def search_with_engine(client: Client, engine: str, query: str) -> dict: + """Search using a specific engine.""" + try: + results = await client.search({"engine": engine, "q": query}) + return { + "engine": engine, + "query": query, + "status": results.get("search_metadata", {}).get("status", "Unknown"), + "organic_count": len(results.get("organic_results", [])), + "total_results": results.get("search_information", {}).get( + "total_results", "N/A" + ), + "success": True, + "results": results.get("organic_results", [])[:3], # First 3 results + } + except SerpApiError as e: + return {"engine": engine, "query": query, "error": str(e), "success": False} + + +async def main(): + """Main example function.""" + # Get API key from environment variable + api_key = os.getenv("SERPAPI_KEY") + if not api_key: + print("Please set SERPAPI_KEY environment variable") + return + + # Create client + client = Client(api_key=api_key, persistent=True) + + # Search query + query = "artificial intelligence" + + # Different search engines to try + engines = ["google", "bing", "yahoo", "duckduckgo", "baidu"] + + try: + print(f"=== Multi-Engine Search: '{query}' ===") + print(f"Searching with {len(engines)} different engines...") + print() + + # Create tasks for concurrent execution + tasks = [search_with_engine(client, engine, query) for engine in engines] + + # Execute all searches concurrently + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Process and display results + for result in results: + if isinstance(result, Exception): + print(f"Unexpected error: {result}") + continue + + print(f"--- {result['engine'].upper()} ---") + + if result["success"]: + print(f"Status: {result['status']}") + print(f"Organic results: {result['organic_count']}") + print(f"Total results: {result['total_results']}") + + if result["results"]: + print("Top results:") + for i, res in enumerate(result["results"], 1): + title = res.get("title", "No title") + link = res.get("link", "No link") + print(f" {i}. {title}") + print(f" {link}") + else: + print("No organic results found") + else: + print(f"Error: {result['error']}") + + print() + + # Compare results across engines + successful_results = [ + r for r in results if isinstance(r, dict) and r.get("success") + ] + if len(successful_results) > 1: + print("=== Comparison Summary ===") + for result in successful_results: + print(f"{result['engine']}: {result['organic_count']} organic results") + + except Exception as e: + print(f"Unexpected error: {e}") + finally: + # Close the client + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ext/serpapi-python.svg b/ext/serpapi-python.svg deleted file mode 100644 index 2b0accc..0000000 --- a/ext/serpapi-python.svg +++ /dev/null @@ -1,1451 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -SerpApi Python Client - diff --git a/pylint.rc b/pylint.rc deleted file mode 100644 index 8cdc70e..0000000 --- a/pylint.rc +++ /dev/null @@ -1,637 +0,0 @@ -[MAIN] - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Clear in-memory caches upon conclusion of linting. Useful if running pylint -# in a server-like mode. -clear-cache-post-run=no - -# Load and enable all available extensions. Use --list-extensions to see a list -# all available extensions. -#enable-all-extensions= - -# In error mode, messages with a category besides ERROR or FATAL are -# suppressed, and no reports are done by default. Error mode is compatible with -# disabling specific errors. -#errors-only= - -# Always return a 0 (non-error) status code, even if lint errors are found. -# This is primarily useful in continuous integration scripts. -#exit-zero= - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-allow-list= - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. (This is an alternative name to extension-pkg-allow-list -# for backward compatibility.) -extension-pkg-whitelist= - -# Return non-zero exit code if any of these messages/categories are detected, -# even if score is above --fail-under value. Syntax same as enable. Messages -# specified are enabled, while categories only check already-enabled messages. -fail-on= - -# Specify a score threshold under which the program will exit with error. -fail-under=10 - -# Interpret the stdin as a python script, whose filename needs to be passed as -# the module_or_package argument. -#from-stdin= - -# Files or directories to be skipped. They should be base names, not paths. -ignore=CVS - -# Add files or directories matching the regular expressions patterns to the -# ignore-list. The regex matches against paths and can be in Posix or Windows -# format. Because '\\' represents the directory delimiter on Windows systems, -# it can't be used as an escape character. -ignore-paths= - -# Files or directories matching the regular expression patterns are skipped. -# The regex matches against base names, not paths. The default value ignores -# Emacs file locks -ignore-patterns=^\.# - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use, and will cap the count on Windows to -# avoid hangs. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Minimum Python version to use for version dependent checks. Will default to -# the version used to run pylint. -py-version=3.11 - -# Discover python modules and packages in the file system subtree. -recursive=no - -# Add paths to the list of the source roots. Supports globbing patterns. The -# source root is an absolute path or a path relative to the current working -# directory used to determine a package namespace for modules located under the -# source root. -source-roots= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# In verbose mode, extra non-checker-related info will be displayed. -#verbose= - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. If left empty, argument names will be checked with the set -# naming style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. If left empty, attribute names will be checked with the set naming -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - victor - kenneth - bart - -# Bad variable names regexes, separated by a comma. If names match any regex, -# they will always be refused -bad-names-rgxs= - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. If left empty, class attribute names will be checked -# with the set naming style. -#class-attribute-rgx= - -# Naming style matching correct class constant names. -class-const-naming-style=UPPER_CASE - -# Regular expression matching correct class constant names. Overrides class- -# const-naming-style. If left empty, class constant names will be checked with -# the set naming style. -#class-const-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. If left empty, class names will be checked with the set naming style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. If left empty, constant names will be checked with the set naming -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. If left empty, function names will be checked with the set -# naming style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Good variable names regexes, separated by a comma. If names match any regex, -# they will always be accepted -good-names-rgxs= - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. If left empty, inline iteration names will be checked -# with the set naming style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. If left empty, method names will be checked with the set naming style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. If left empty, module names will be checked with the set naming style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Regular expression matching correct type alias names. If left empty, type -# alias names will be checked with the set naming style. -#typealias-rgx= - -# Regular expression matching correct type variable names. If left empty, type -# variable names will be checked with the set naming style. -#typevar-rgx= - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. If left empty, variable names will be checked with the set -# naming style. -#variable-rgx= - - -[CLASSES] - -# Warn about protected attribute access inside special methods -check-protected-access-in-special-methods=no - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - asyncSetUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# List of regular expressions of class ancestor names to ignore when counting -# public methods (see R0903) -exclude-too-few-public-methods= - -# List of qualified class names to ignore when counting class parents (see -# R0901) -ignored-parents= - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when caught. -overgeneral-exceptions=builtins.BaseException,builtins.Exception - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow explicit reexports by alias from a package __init__. -allow-reexport-from-package=no - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules= - -# Output a graph (.gv or any supported image format) of external dependencies -# to the given file (report RP0402 must not be disabled). -ext-import-graph= - -# Output a graph (.gv or any supported image format) of all (i.e. internal and -# external) dependencies to the given file (report RP0402 must not be -# disabled). -import-graph= - -# Output a graph (.gv or any supported image format) of internal dependencies -# to the given file (report RP0402 must not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[LOGGING] - -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, -# UNDEFINED. -confidence=HIGH, - CONTROL_FLOW, - INFERENCE, - INFERENCE_FAILURE, - UNDEFINED - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then re-enable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead - unnecessary-pass - invalid-name - - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[METHOD_ARGS] - -# List of qualified names (i.e., library.method) which require a timeout -# parameter e.g. 'requests.api.get,requests.api.post' -timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - -# Regular expression of note tags to take in consideration. -notes-rgx= - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit,argparse.parse_error - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'fatal', 'error', 'warning', 'refactor', -# 'convention', and 'info' which contain the number of messages in each -# category, as well as 'statement' which is the total number of statements -# analyzed. This score is used by the global evaluation report (RP0004). -evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -#output-format= - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[SIMILARITIES] - -# Comments are removed from the similarity computation -ignore-comments=yes - -# Docstrings are removed from the similarity computation -ignore-docstrings=yes - -# Imports are removed from the similarity computation -ignore-imports=yes - -# Signatures are removed from the similarity computation -ignore-signatures=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. No available dictionaries : You need to install -# both the python package and the system dependency for enchant to work.. -spelling-dict= - -# List of comma separated words that should be considered directives if they -# appear at the beginning of a comment and should not be checked. -spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no - -# This flag controls whether the implicit-str-concat should generate a warning -# on implicit string concatenation in sequences defined over several lines. -check-str-concat-over-line-jumps=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of symbolic message names to ignore for Mixin members. -ignored-checks-for-mixins=no-member, - not-async-context-manager, - not-context-manager, - attribute-defined-outside-init - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# Regex pattern to define which classes are considered mixins. -mixin-class-rgx=.*[Mm]ixin - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of names allowed to shadow builtins -allowed-redefined-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/pyproject.toml b/pyproject.toml index 73c7799..987d50e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,66 +1,128 @@ [build-system] -requires = ["setuptools>=61", "wheel"] +requires = ["setuptools>=61.0", "wheel"] build-backend = "setuptools.build_meta" [project] name = "serpapi" -dynamic = ["version"] -description = "The official Python client for SerpApi.com." +version = "0.2.0" +description = "Official Python library for SerpApi.com - Search Engine Results API" readme = "README.md" -license = { text = "MIT" } -authors = [{ name = "SerpApi", email = "support@serpapi.com" }] -requires-python = ">=3.6" -dependencies = ["requests"] -keywords = [ - "scrape", "serp", "api", "serpapi", "scraping", "json", "search", - "localized", "rank", "google", "bing", "baidu", "yandex", "yahoo", - "ebay", "scale", "datamining", "training", "machine", "ml", - "youtube", "naver", "walmart", "apple", "store", "app", +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "victor benarbia", email = "victor@serpapi.com"}, + {name = "Julien Khaleghy"} ] +keywords = ["serpapi", "search", "google", "bing", "yahoo", "scraping", "api"] classifiers = [ - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.15", - "Programming Language :: Python :: Implementation :: CPython", - "Natural Language :: English", - "Topic :: Utilities", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Internet :: WWW/HTTP :: Indexing/Search", - "Topic :: Software Development :: Libraries :: Python Modules", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "aiohttp>=3.9.0,<4", + "yarl>=1.9", + "multidict>=6", ] [project.optional-dependencies] -color = ["pygments"] -test = ["pytest"] -docs = [ - "sphinx>=8.2,<9; python_version >= '3.11'", - "myst-parser>=4,<5; python_version >= '3.11'", - "sphinx-design>=0.6,<1; python_version >= '3.11'", - "sphinx-rtd-theme>=3.1,<4; python_version >= '3.11'", +dev = [ + "pytest>=7", + "pytest-asyncio>=0.23", + "mypy>=1.8", + "pytest-cov>=4.1", ] [project.urls] Homepage = "https://github.com/serpapi/serpapi-python" -Source = "https://github.com/serpapi/serpapi-python" -Documentation = "https://serpapi-python.readthedocs.io/en/latest/" - -[tool.setuptools.dynamic] -version = { attr = "serpapi.__version__.__version__" } +Documentation = "https://serpapi.com" +Repository = "https://github.com/serpapi/serpapi-python" +Issues = "https://github.com/serpapi/serpapi-python/issues" [tool.setuptools.packages.find] -include = ["serpapi", "serpapi.*"] -exclude = ["tests", "tests.*"] +where = ["."] +include = ["serpapi*"] [tool.pytest.ini_options] -testpaths = ["tests", "serpapi"] -addopts = "--doctest-modules" -doctest_optionflags = ["ELLIPSIS", "NORMALIZE_WHITESPACE"] +asyncio_mode = "auto" +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[tool.coverage.run] +source = ["serpapi"] +omit = ["tests/*", "*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[tool.black] +line-length = 88 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 88 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +skip_glob = ["*/migrations/*"] + +[dependency-groups] +dev = [ + "mypy>=1.18.1", + "pytest>=8.4.2", + "pytest-asyncio>=1.2.0", + "pytest-cov>=7.0.0", + "black>=24.0.0", + "isort>=5.13.0", +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5f1a014 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# Core dependencies +aiohttp>=3.9.0,<4 +yarl>=1.9 +multidict>=6 + +# Development dependencies (optional) +pytest>=7 +pytest-asyncio>=0.23 +mypy>=1.8 +pytest-cov>=4.1 diff --git a/scripts/check_docs_revision.py b/scripts/check_docs_revision.py deleted file mode 100644 index 16af8b0..0000000 --- a/scripts/check_docs_revision.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -import os -import re -import shlex -import subprocess -import sys -import time - - -CONTEXT_VARIABLE = "DOCS_CI_REVISION" - - -def check_revision(environ, commit, now=None): - if environ.get("READTHEDOCS_VERSION_TYPE") == "external": - return - try: - value = environ[CONTEXT_VARIABLE] - # RTD stores custom environment values with shlex.quote. - if value.startswith("'"): - value, = shlex.split(value) - context = json.loads(value) - expected = context["commit"] - versions = context["versions"] - expires = context["expires_at"] - if not re.fullmatch(r"[0-9a-f]{40}", expected): - raise ValueError - if not isinstance(versions, list) or not versions: - raise ValueError - if not isinstance(expires, (int, float)): - raise ValueError - except (KeyError, TypeError, ValueError): - raise RuntimeError( - "No valid CI revision was supplied. Publish through the Documentation workflow." - ) from None - if (time.time() if now is None else now) >= expires: - raise RuntimeError("The CI revision expired. Rerun the Documentation workflow.") - name = environ.get("READTHEDOCS_VERSION_NAME") or environ.get("READTHEDOCS_VERSION") - if name not in versions: - raise RuntimeError("CI did not request publication of this documentation version.") - if commit != expected: - raise RuntimeError( - "RTD checked out a different commit than the one tested by CI. " - "Rerun the Documentation workflow on the current branch or tag." - ) - - -def main(): - commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() - check_revision(os.environ, commit) - print("Documentation revision check passed.") - - -if __name__ == "__main__": - try: - main() - except RuntimeError as exc: - print(str(exc), file=sys.stderr) - sys.exit(1) diff --git a/scripts/publish_docs.py b/scripts/publish_docs.py deleted file mode 100644 index 708d0e7..0000000 --- a/scripts/publish_docs.py +++ /dev/null @@ -1,209 +0,0 @@ -import json -import os -import re -import subprocess -import sys -import time -from urllib.error import HTTPError, URLError -from urllib.parse import quote, urlencode, urljoin -from urllib.request import HTTPRedirectHandler, Request, build_opener - -from scripts.check_docs_revision import CONTEXT_VARIABLE - - -PROJECT = "serpapi-python" -API_ROOT = "https://app.readthedocs.org/api/v3/" -POLL_SECONDS = 10 -BUILD_TIMEOUT = 1200 - - -class APIError(RuntimeError): - def __init__(self, status): - self.status = status - super().__init__(f"Read the Docs API returned HTTP {status}.") - - -class NoRedirect(HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -class ReadTheDocs: - def __init__(self, token): - self.token = token - self.base = f"{API_ROOT}projects/{PROJECT}/" - self.open = build_opener(NoRedirect()).open - - def request(self, method, path, data=None): - url = urljoin(self.base, path) - if not url.startswith(self.base): - raise RuntimeError("Refusing to send the RTD token outside this project's API.") - body = None if data is None else json.dumps(data).encode() - request = Request(url, data=body, method=method, headers={ - "Authorization": f"Token {self.token}", - "Content-Type": "application/json", - "User-Agent": "serpapi-python-docs-publisher", - }) - try: - with self.open(request, timeout=30) as response: - payload = response.read() - except HTTPError as exc: - raise APIError(exc.code) from None - except (URLError, TimeoutError): - raise RuntimeError("Could not reach the Read the Docs API.") from None - try: - return json.loads(payload) if payload else None - except ValueError: - raise RuntimeError("Read the Docs returned an invalid API response.") from None - - def items(self, path): - while path: - page = self.request("GET", path) - yield from page["results"] - path = page.get("next") - - -def wait_for(check, message, timeout=BUILD_TIMEOUT): - deadline = time.monotonic() + timeout - while True: - result = check() - if result: - return result - if time.monotonic() >= deadline: - raise RuntimeError(message) - time.sleep(POLL_SECONDS) - - -def version_path(slug): - return f"versions/{quote(slug, safe='')}/" - - -def find_version(api, ref): - if ref == "refs/heads/master": - return api.request("GET", version_path("latest")) - query = urlencode({"type": "tag", "verbose_name": ref[len("refs/tags/"):]}) - versions = list(api.items(f"versions/?{query}")) - if len(versions) > 1: - raise RuntimeError("More than one RTD version matches the release tag.") - return versions[0] if versions else None - - -def wait_for_build(api, build, commit): - build_id = build["id"] - url = f"https://app.readthedocs.org/projects/{PROJECT}/builds/{build_id}/" - print(f"Waiting for RTD build: {url}", flush=True) - - def finished(): - current = api.request("GET", f"builds/{build_id}/") - state = current["state"]["code"] - if state not in ("finished", "cancelled"): - return None - if not current.get("success"): - raise RuntimeError(f"RTD build failed or was cancelled. See {url}") - if current.get("commit") != commit: - raise RuntimeError(f"RTD built a different commit. See {url}") - return current - - return wait_for(finished, f"Timed out waiting for RTD. See {url}") - - -def newest_build(api, version, after): - for build in api.items("builds/"): - if build["id"] <= after: - break - if build["version"] == version: - return build - return None - - -def publish_version(api, version, commit, after=None): - path = version_path(version["slug"]) - # Syncing versions can queue stable and release builds without an automation rule. - build = newest_build(api, version["slug"], after) if after is not None else None - if build: - return wait_for_build(api, build, commit) - if not version["active"]: - previous = api.request("GET", "builds/?limit=1")["results"] - last_id = previous[0]["id"] if previous else 0 - # Activating a version already queues a build; do not queue a second one. - api.request("PATCH", path, {"active": True, "hidden": False}) - - def activated_build(): - return newest_build(api, version["slug"], last_id) - - build = wait_for(activated_build, "RTD did not queue the activated version.", 120) - else: - build = api.request("POST", path + "builds/")["build"] - return wait_for_build(api, build, commit) - - -def clear_context(api): - for variable in list(api.items("environmentvariables/")): - if variable["name"] == CONTEXT_VARIABLE: - api.request("DELETE", f"environmentvariables/{variable['pk']}/") - - -def publish(api, ref, commit): - if ref != "refs/heads/master" and not ref.startswith("refs/tags/v"): - raise RuntimeError("Only master and v-prefixed release tags can publish documentation.") - if not re.fullmatch(r"[0-9a-f]{40}", commit): - raise RuntimeError("Expected a full Git commit hash.") - project = api.request("GET", "") - if project["default_branch"] != "master": - raise RuntimeError("Set the RTD project's Default branch to master first.") - - # Serialize in Actions, and let any previous RTD build finish before changing its context. - wait_for( - lambda: not api.request("GET", "builds/?running=true")["results"], - "Existing RTD builds did not finish. Check the RTD dashboard before retrying.", - ) - clear_context(api) - previous = api.request("GET", "builds/?limit=1")["results"] - last_id = previous[0]["id"] if previous else 0 - versions = ["latest"] if ref == "refs/heads/master" else [ref[len("refs/tags/"):], "stable", "latest"] - context = { - "commit": commit, - "versions": versions, - "expires_at": time.time() + 3600, - } - variable = api.request("POST", "environmentvariables/", { - "name": CONTEXT_VARIABLE, "value": json.dumps(context), "public": False, - }) - try: - api.request("POST", "sync-versions/") - version = wait_for(lambda: find_version(api, ref), "RTD did not discover the Git tag.", 120) - publish_version(api, version, commit, after=last_id) - if ref.startswith("refs/tags/"): - try: - stable = api.request("GET", version_path("stable")) - except APIError as exc: - if exc.status != 404: - raise - stable = None - if stable and stable.get("ref") == ref[len("refs/tags/"):]: - publish_version(api, stable, commit, after=last_id) - latest = api.request("GET", version_path("latest")) - publish_version(api, latest, commit, after=last_id) - print(f"Published documentation for {ref} at {commit}.", flush=True) - finally: - api.request("DELETE", f"environmentvariables/{variable['pk']}/") - - -def main(): - token = os.environ.get("RTD_API_TOKEN") - if not token: - raise RuntimeError("Add RTD_API_TOKEN to the docs GitHub environment before publishing.") - if os.environ.get("GITHUB_EVENT_NAME") not in ("push", "workflow_dispatch"): - raise RuntimeError("PR runs cannot publish documentation.") - commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() - if commit != os.environ.get("GITHUB_SHA"): - raise RuntimeError("The publishing checkout does not match the commit tested by CI.") - publish(ReadTheDocs(token), os.environ.get("GITHUB_REF", ""), commit) - - -if __name__ == "__main__": - try: - main() - except RuntimeError as exc: - print(str(exc), file=sys.stderr) - sys.exit(1) diff --git a/serpapi/__init__.py b/serpapi/__init__.py index 39763c2..7a1d707 100644 --- a/serpapi/__init__.py +++ b/serpapi/__init__.py @@ -1,4 +1,19 @@ -from .__version__ import __version__ +""" +SerpApi Python Client Library -from .core import * -from .exceptions import * +Official Python client for SerpApi.com - Search Engine Results API. +Integrate search data into your AI workflow, RAG / fine-tuning, or Python application. + +Supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more. +""" + +from .error import SerpApiError +from .version import __version__ + +# Import Client only when aiohttp is available +try: + from .client import Client + + __all__ = ["Client", "SerpApiError", "__version__"] +except ImportError: + __all__ = ["SerpApiError", "__version__"] diff --git a/serpapi/__version__.py b/serpapi/__version__.py deleted file mode 100644 index c68196d..0000000 --- a/serpapi/__version__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.2.0" diff --git a/serpapi/client.py b/serpapi/client.py new file mode 100644 index 0000000..50ef8a6 --- /dev/null +++ b/serpapi/client.py @@ -0,0 +1,339 @@ +""" +Client implementation for SerpApi.com + +Powered by aiohttp for async HTTP requests and persistent connections. +""" + +import asyncio +import json +import os +from typing import Any, Dict, List, Literal, Optional, Union, overload +from urllib.parse import urlencode + +import aiohttp +from aiohttp import ClientSession, ClientTimeout + +from .error import SerpApiError +from .version import __version__ + + +class Client: + """ + Client for SerpApi.com + + Features: + - Async non-blocking search + - Persistent HTTP connections + - Search API + - Location API + - Account API + - Search Archive API + """ + + # Backend service URL + BACKEND = "serpapi.com" + + def __init__( + self, + api_key: Optional[str] = None, + engine: str = "google", + persistent: bool = True, + async_mode: bool = False, + timeout: int = 120, + symbolize_names: bool = True, + **kwargs: Any, + ): + """ + Initialize SerpApi client. + + Args: + api_key: User secret API key. If None, will try to get from SERPAPI_KEY env var. + engine: Default search engine selection. + persistent: Keep socket connection open for faster response times (2x faster). + async_mode: Enable async mode for non-blocking operations. + timeout: HTTP request timeout in seconds. + symbolize_names: Convert JSON keys to symbols (not applicable in Python, kept for compatibility). + **kwargs: Additional parameters to store as default parameters. + + Raises: + SerpApiError: If parameters are invalid. + """ + if api_key is None: + api_key = os.getenv("SERPAPI_KEY") + if api_key is None: + raise SerpApiError( + "API key is required. Set api_key parameter or SERPAPI_KEY environment variable." + ) + + # Store configuration + self._timeout = timeout + self._persistent = persistent + self._async_mode = async_mode + self._symbolize_names = symbolize_names + + # Set default query parameters + self._params = { + "api_key": api_key, + "engine": engine, + "source": f"serpapi-python:{__version__}", + **kwargs, + } + + # HTTP client session (will be created when needed) + self._session: Optional[ClientSession] = None + self._session_lock = asyncio.Lock() + + @property + def timeout(self) -> int: + """Get HTTP timeout in seconds.""" + return self._timeout + + @property + def persistent(self) -> bool: + """Check if persistent connections are enabled.""" + return self._persistent + + @property + def engine(self) -> str: + """Get default search engine.""" + return str(self._params.get("engine", "google")) + + @property + def api_key(self) -> str: + """Get API key.""" + return str(self._params.get("api_key", "")) + + async def _get_session(self) -> ClientSession: + """Get or create HTTP session.""" + if not self._persistent or self._session is None or self._session.closed: + async with self._session_lock: + if ( + not self._persistent + or self._session is None + or self._session.closed + ): + timeout = ClientTimeout(total=self._timeout) + connector = aiohttp.TCPConnector(limit=100, limit_per_host=30) + self._session = ClientSession( + timeout=timeout, + connector=connector, + base_url=f"https://{self.BACKEND}", + ) + return self._session + + def _merge_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """ + Merge runtime parameters with default parameters. + + Args: + params: Runtime parameters to merge. + + Returns: + Merged parameters after cleanup. + + Raises: + SerpApiError: If params is not a dictionary. + """ + if not isinstance(params, dict): + raise SerpApiError(f"params must be dict, not: {type(params)}") + + # Merge default params with custom params + merged = self._params.copy() + merged.update(params) + + # Remove client-specific configuration + merged.pop("symbolize_names", None) + + # Remove None values + return {k: v for k, v in merged.items() if v is not None} + + @overload + async def _make_request( + self, + endpoint: str, + params: Dict[str, Any], + response_format: Literal["json"] = "json", + ) -> Dict[str, Any]: ... + + @overload + async def _make_request( + self, endpoint: str, params: Dict[str, Any], response_format: Literal["html"] + ) -> str: ... + + async def _make_request( + self, endpoint: str, params: Dict[str, Any], response_format: str = "json" + ) -> Union[Dict[str, Any], str]: + """ + Make HTTP request to SerpApi backend. + + Args: + endpoint: API endpoint path. + params: Request parameters. + response_format: Response format ('json' or 'html'). + + Returns: + Response data as dict (JSON) or str (HTML). + + Raises: + SerpApiError: If request fails or response is invalid. + """ + session = await self._get_session() + query_params = self._merge_params(params) + + try: + async with session.get(endpoint, params=query_params) as response: + if response.status != 200: + try: + data = await response.json() + if isinstance(data, dict) and "error" in data: + raise SerpApiError( + f"SerpApi error: {data['error']}" + f" from url: https://{self.BACKEND}{endpoint} with status: {response.status}" + ) + except json.JSONDecodeError: + err = await response.text() + raise SerpApiError( + f"HTTP request failed with error: {err}" + f" from url: https://{self.BACKEND}{endpoint} with status: {response.status}" + ) + + if response_format == "json": + try: + data = await response.json() + if isinstance(data, dict) and "error" in data: + raise SerpApiError(f"SerpApi error: {data['error']}") + return data # type: ignore + except json.JSONDecodeError: + text = await response.text() + raise SerpApiError(f"Invalid JSON response: {text}") + + elif response_format == "html": + return await response.text() + + else: + raise SerpApiError( + f"Unsupported response format: {response_format}" + ) + + except aiohttp.ClientError as e: + raise SerpApiError(f"HTTP client error: {e}") + + async def search(self, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """ + Perform a search using SerpApi.com + + Args: + params: Search parameters including engine, query, etc. + + Returns: + Search results as a dictionary. + """ + if params is None: + params = {} + return await self._make_request("/search", params, "json") + + async def html(self, params: Optional[Dict[str, Any]] = None) -> str: + """ + Perform a search and return raw HTML. + + Useful for training AI models, RAG, debugging, or custom parsing. + + Args: + params: Search parameters. + + Returns: + Raw HTML search results directly from the search engine. + """ + if params is None: + params = {} + return await self._make_request("/search", params, "html") + + async def location( + self, params: Optional[Dict[str, Any]] = None + ) -> List[Dict[str, Any]]: + """ + Get location suggestions using Location API. + + Args: + params: Location parameters including 'q' (query) and 'limit'. + + Returns: + List of matching locations. + """ + if params is None: + params = {} + result = await self._make_request("/locations.json", params, "json") + return result # type: ignore + + async def search_archive( + self, search_id: Union[str, int], format_type: str = "json" + ) -> Union[Dict[str, Any], str]: + """ + Retrieve search result from the Search Archive API. + + Args: + search_id: Search ID from original search results. + format_type: Response format ('json' or 'html'). + + Returns: + Archived search results as dict (JSON) or str (HTML). + + Raises: + SerpApiError: If format_type is invalid. + """ + if format_type not in ["json", "html"]: + raise SerpApiError("format_type must be json or html") + + empty_params: Dict[str, Any] = {} + if format_type == "json": + json_result: Dict[str, Any] = await self._make_request( + f"/searches/{search_id}.{format_type}", empty_params, "json" + ) + return json_result + elif format_type == "html": + html_result: str = await self._make_request( + f"/searches/{search_id}.{format_type}", empty_params, "html" + ) + return html_result + else: + raise SerpApiError("format_type must be json or html") + + async def account(self, api_key: Optional[str] = None) -> Dict[str, Any]: + """ + Get account information using Account API. + + Args: + api_key: API key (optional if already provided to constructor). + + Returns: + Account information dictionary. + """ + params = {"api_key": api_key} if api_key else {} + return await self._make_request("/account", params, "json") + + async def close(self) -> None: + """Close HTTP session if persistent connections are enabled.""" + if self._session and not self._session.closed: + await self._session.close() + + async def __aenter__(self) -> "Client": + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Async context manager exit.""" + await self.close() + + def __del__(self) -> None: + """Destructor to ensure session is closed.""" + if hasattr(self, "_session") and self._session and not self._session.closed: + # Schedule the session close in the event loop + try: + loop = asyncio.get_event_loop() + if loop.is_running(): + loop.create_task(self._session.close()) + else: + loop.run_until_complete(self._session.close()) + except RuntimeError: + # No event loop running, can't close session + pass diff --git a/serpapi/core.py b/serpapi/core.py deleted file mode 100644 index f00f7cd..0000000 --- a/serpapi/core.py +++ /dev/null @@ -1,223 +0,0 @@ -import io -import os - -from .http import HTTPClient -from .exceptions import SearchIDNotProvided -from .models import SerpResults - - -class Client(HTTPClient): - """A class that handles API requests to SerpApi in a user-friendly manner. - - Store your API key in an environment variable, then create a client with - ``serpapi.Client(api_key=os.environ["SERPAPI_KEY"])``. - - :param api_key: The API Key to use for SerpApi.com. - :param timeout: The default timeout to use for requests. - """ - - DASHBOARD_URL = "https://serpapi.com/dashboard" - - def __init__(self, *, api_key=None, timeout=None): - super().__init__(api_key=api_key, timeout=timeout) - - def __repr__(self): - return "" - - def search(self, params: dict = None, **kwargs): - """Fetch a page of results from SerpApi. - - Returns a ``serpapi.SerpResults`` object for JSON responses, or text - when ``output="html"`` or ``output="md"`` is requested. Prefer passing SerpApi engine - parameters as keyword arguments. A parameter dictionary is also accepted - when your code already has parameters in a mapping. - - - :param params: Optional mapping of SerpApi search parameters such as ``engine``, ``q``, ``location``, and ``output``. - :param kwargs: Additional SerpApi parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. - - - **Learn more**: https://serpapi.com/search-api - """ - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - r = self.request("GET", "/search", params=params, **request_kwargs) - - return SerpResults.from_http_response(r, client=self) - - def extract(self, params: dict = None, **kwargs): - """Extract content from a public URL using the Extract API. - - Accepts a parameter dictionary or keyword arguments, like :meth:`search`. - Sets ``engine=extract`` and returns a :class:`SerpResults ` object for JSON, or text for HTML and Markdown. - - .. code-block:: python - - results = client.extract(url="https://example.com/") - markdown = client.extract(url="https://example.com/", output="md") - results = client.extract({"url": "https://example.com/", "timeout": 30000}, timeout=60) - - :param params: Optional mapping of Extract API parameters, including the required ``url``. Put ``timeout`` here to set the API fetch timeout in milliseconds. - :param kwargs: Additional API parameters or HTTP request options, as in :meth:`search`. The ``timeout`` keyword controls the HTTP client timeout in seconds. - - ``mode`` accepts ``scrape`` (default), ``map``, or ``crawl``. ``output`` accepts ``json`` (default), ``html``, or ``md``. - JSON includes page content in the top-level ``markdown`` field by default, - or in each ``pages`` entry for crawls. Set ``include_markdown=False`` to omit it. - For a scraped page, ``output="md"`` returns the page Markdown without search metadata. - - Crawl mode can consume one search credit per successfully extracted page. Use ``max_pages`` to limit the number of pages. - """ - params = dict(params) if params is not None else {} - kwargs["engine"] = "extract" - return self.search(params, **kwargs) - - def search_archive(self, params: dict = None, **kwargs): - """Get a result from the SerpApi Search Archive API. - - :param params: Archive parameters. Must include ``search_id``. ``output`` accepts ``json`` (default), ``html``, or ``md``. - :param kwargs: Additional archive parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. - - **Learn more**: https://serpapi.com/search-archive-api - """ - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - try: - search_id = params["search_id"] - except KeyError: - raise SearchIDNotProvided( - f"Please provide 'search_id', found here: { self.DASHBOARD_URL }" - ) - - r = self.request("GET", f"/searches/{ search_id }", params=params, **request_kwargs) - return SerpResults.from_http_response(r, client=self) - - def upload_image(self, image, **kwargs): - """Upload an image to SerpApi's Image API. - - ``image`` can be a filesystem path or an open binary file object. The - returned dictionary contains an ``image_id`` that can be passed to - :meth:`search` for engines that accept uploaded images, such as Google - Lens. - - :param image: a path or open binary file object containing a JPG/JPEG, - PNG, or WebP image no larger than 500 KB. - :param api_key: the API Key to use for SerpApi.com. - :param **: any additional multipart form fields to pass to the API. - - **Learn more**: https://serpapi.com/image-api - """ - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - data = kwargs - if "api_key" not in data: - data["api_key"] = self.api_key - - image_file = None - try: - if isinstance(image, (str, os.PathLike)): - image_file = open(image, "rb") - image = image_file - elif isinstance(image, io.TextIOBase): - raise TypeError( - "image file must be opened in binary mode, e.g. open(path, 'rb')" - ) - - r = self.request( - "POST", - "/image", - params={}, - data=data, - files={"image": image}, - **request_kwargs, - ) - return r.json() - finally: - if image_file is not None: - image_file.close() - - def locations(self, params: dict = None, **kwargs): - """Get a list of supported Google locations. - - - :param params: Location API parameters such as ``q`` and ``limit``. - :param kwargs: Additional location parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. - - **Learn more**: https://serpapi.com/locations-api - """ - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - r = self.request( - "GET", - "/locations.json", - params=params, - assert_200=True, - **request_kwargs, - ) - return r.json() - - def account(self, params: dict = None, **kwargs): - """Get SerpApi account information. - - :param params: Account API parameters. - :param kwargs: Additional account parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. - - **Learn more**: https://serpapi.com/account-api - """ - - if params is None: - params = {} - - # These are arguments that should be passed to the underlying requests.request call. - request_kwargs = {} - for key in ["timeout", "proxies", "verify", "stream", "cert"]: - if key in kwargs: - request_kwargs[key] = kwargs.pop(key) - - if kwargs: - params.update(kwargs) - - r = self.request("GET", "/account.json", params=params, assert_200=True, **request_kwargs) - return r.json() - - -# An un-authenticated client instance. -_client = Client() -search = _client.search -extract = _client.extract -search_archive = _client.search_archive -upload_image = _client.upload_image -locations = _client.locations -account = _client.account diff --git a/serpapi/error.py b/serpapi/error.py new file mode 100644 index 0000000..8d3383c --- /dev/null +++ b/serpapi/error.py @@ -0,0 +1,20 @@ +""" +SerpApi error handling module. +""" + + +class SerpApiError(Exception): + """ + SerpApiError wraps any errors related to the SerpApi client. + + Handles the following types of errors: + - HTTP response errors from SerpApi.com + - Missing API key + - Credit limit exceeded + - Incorrect query parameters + - JSON parsing errors + - Network timeouts + - And more... + """ + + pass diff --git a/serpapi/exceptions.py b/serpapi/exceptions.py deleted file mode 100644 index c268c4d..0000000 --- a/serpapi/exceptions.py +++ /dev/null @@ -1,51 +0,0 @@ -import requests - - -class SerpApiError(Exception): - """Base class for exceptions in this module.""" - - pass - - -class APIKeyNotProvided(ValueError, SerpApiError): - """API key is not provided.""" - - pass - - -class SearchIDNotProvided(ValueError, SerpApiError): - """Search ID is not provided.""" - - pass - - -class HTTPError(requests.exceptions.HTTPError, SerpApiError): - """HTTP Error.""" - - def __init__(self, original_exception): - if (isinstance(original_exception, requests.exceptions.HTTPError)): - http_error_exception: requests.exceptions.HTTPError = original_exception - - self.status_code = http_error_exception.response.status_code - try: - self.error = http_error_exception.response.json().get("error", None) - except requests.exceptions.JSONDecodeError: - self.error = None - else: - self.status_code = -1 - self.error = None - - super().__init__(*original_exception.args, response=getattr(original_exception, 'response', None), request=getattr(original_exception, 'request', None)) - - - -class HTTPConnectionError(HTTPError, requests.exceptions.ConnectionError, SerpApiError): - """Connection Error.""" - - pass - - -class TimeoutError(requests.exceptions.Timeout, SerpApiError): - """Timeout Error.""" - - pass diff --git a/serpapi/http.py b/serpapi/http.py deleted file mode 100644 index 4ea1a33..0000000 --- a/serpapi/http.py +++ /dev/null @@ -1,71 +0,0 @@ -import requests - -from .exceptions import ( - HTTPError, - HTTPConnectionError, - TimeoutError, -) -from .__version__ import __version__ - - -class HTTPClient: - """This class handles outgoing HTTP requests to SerpApi.com.""" - - BASE_DOMAIN = "https://serpapi.com" - USER_AGENT = f"serpapi-python, v{__version__}" - - def __init__(self, *, api_key=None, timeout=None): - # Used to authenticate requests. - # TODO: do we want to support the environment variable? Seems like a security risk. - self.api_key = api_key - self.timeout = timeout - self.session = requests.Session() - - def request(self, method, path, params, *, assert_200=True, **kwargs): - # Inject the API Key into the params. - request_data = kwargs.get("data") - api_key_in_data = isinstance(request_data, dict) and "api_key" in request_data - if "api_key" not in params and not api_key_in_data: - params["api_key"] = self.api_key - - # Build the URL, as needed. - if not path.startswith("http"): - url = self.BASE_DOMAIN + path - else: - url = path - - # Make the HTTP request. - try: - headers = {"User-Agent": self.USER_AGENT} - - # Use the default timeout if one was provided to the client. - if self.timeout and "timeout" not in kwargs: - kwargs["timeout"] = self.timeout - - r = self.session.request( - method=method, url=url, params=params, headers=headers, **kwargs - ) - - except requests.exceptions.ConnectionError as e: - raise HTTPConnectionError(e) - except requests.exceptions.Timeout as e: - raise TimeoutError(e) - - # Raise an exception if the status code is not 200. - if assert_200: - try: - raise_for_status(r) - except requests.exceptions.HTTPError as e: - raise HTTPError(e) - - return r - - -def raise_for_status(r): - """Raise an exception if the status code is not 200.""" - # TODO: put custom behavior in here for various status codes. - - try: - r.raise_for_status() - except requests.exceptions.HTTPError as e: - raise HTTPError(e) diff --git a/serpapi/models.py b/serpapi/models.py deleted file mode 100644 index 3e8a982..0000000 --- a/serpapi/models.py +++ /dev/null @@ -1,101 +0,0 @@ -import json - -from collections import UserDict - -from .textui import prettify_json - - -class SerpResults(UserDict): - """A dictionary-like object that represents the results of a SerpApi request. - - An instance of this class is returned if the response is a valid JSON object. - It can be used like a dictionary, but also has some additional methods. - """ - - def __init__(self, data, *, client): - super().__init__(data) - self.client = client - - def __getstate__(self): - return self.data - - def __setstate__(self, state): - self.data = state - - def __repr__(self): - """The visual representation of the data, which is pretty printed, for - ease of use. - """ - - return prettify_json(json.dumps(self.data, indent=4)) - - def as_dict(self): - """Returns the data as a standard Python dictionary. - - This can be useful when passing results to libraries that expect a - plain ``dict``. - """ - - return self.data.copy() - - @property - def next_page_url(self): - """The URL of the next page of results, if any.""" - - serpapi_pagination = self.data.get("serpapi_pagination") - - if serpapi_pagination: - return serpapi_pagination.get("next") - - def next_page(self): - """Return the next page of results, if any.""" - - if self.next_page_url: - # Include support for the API key, as it is not included in the next page URL. - params = {"api_key": self.client.api_key} - - r = self.client.request("GET", path=self.next_page_url, params=params) - return SerpResults.from_http_response(r, client=self.client) - - def yield_pages(self, max_pages=1_000): - """A generator that ``yield`` s the next ``n`` pages of search results, if any. - - :param max_pages: limit the number of pages yielded to ``n``. - """ - - current_page_count = 0 - - current_page = self - while current_page and current_page_count < max_pages: - yield current_page - current_page_count += 1 - if current_page_count >= max_pages: - break - if current_page.next_page_url: - current_page = current_page.next_page() - else: - break - - - @classmethod - def from_http_response(cls, r, *, client=None): - """Construct a SerpResults object from an HTTP response. - - :param r: The HTTP response to parse. - :param client: the Client instance which was used to send this request. - - An instance of this class is returned if the response is a valid JSON object. - Otherwise, the raw text (as a properly decoded unicode string) is returned. - """ - - content_type = r.headers.get("Content-Type", "").split(";", 1)[0].lower() - if content_type in {"text/html", "text/markdown"}: - return r.text - - try: - cls = cls(r.json(), client=client) - - return cls - except ValueError: - # If the response is not JSON, return the raw text. - return r.text diff --git a/serpapi/textui.py b/serpapi/textui.py deleted file mode 100644 index d82b788..0000000 --- a/serpapi/textui.py +++ /dev/null @@ -1,16 +0,0 @@ -try: - import pygments - from pygments import highlight, lexers, formatters -except ImportError: - pygments = None - - -def prettify_json(s): - if pygments: - return highlight( - s, - lexers.JsonLexer(), - formatters.TerminalFormatter(), - ) - else: - return s diff --git a/serpapi/utils.py b/serpapi/utils.py deleted file mode 100644 index 4a94510..0000000 --- a/serpapi/utils.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - - -def api_key_from_environment(): - return os.getenv("SERP_API_KEY") diff --git a/serpapi/version.py b/serpapi/version.py new file mode 100644 index 0000000..eca3e8f --- /dev/null +++ b/serpapi/version.py @@ -0,0 +1,5 @@ +""" +Version information for SerpApi Python client. +""" + +__version__ = "0.2.0" diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..8c4ba8d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Test package for SerpApi Python client diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index fb93fcf..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,55 +0,0 @@ -import os - -import pytest - -import serpapi - -os.environ["CI"] = "1" - - -def pytest_addoption(parser): - parser.addoption( - "--require-docs-key", - action="store_true", - help="Fail instead of skipping live docs examples when the API key is missing", - ) - - -def pytest_sessionstart(session): - if session.config.getoption("--require-docs-key") and not ( - os.environ.get("SERPAPI_KEY") or os.environ.get("API_KEY") - ): - raise pytest.UsageError("Live docs checks require SERPAPI_KEY or API_KEY") - - -@pytest.fixture -def api_key(): - return os.environ["API_KEY"] - - -@pytest.fixture -def client(api_key): - return serpapi.Client(api_key=api_key) - - -@pytest.fixture -def invalid_key_client(api_key): - return serpapi.Client(api_key="bunk-key") - - -@pytest.fixture -def coffee_params(): - return {"q": "Coffee"} - - -@pytest.fixture -def coffee_search(client, coffee_params): - return client.search(**coffee_params) - - -@pytest.fixture -def coffee_search_html(client, coffee_params): - params = coffee_params.copy() - params["output"] = "html" - - return client.search(**params) diff --git a/tests/docs_example_support.py b/tests/docs_example_support.py deleted file mode 100644 index 1d48679..0000000 --- a/tests/docs_example_support.py +++ /dev/null @@ -1,172 +0,0 @@ -import json -import os -from pathlib import Path -import re -import shutil -import signal -import subprocess -import sys -import threading -from urllib.parse import parse_qs, urlsplit - -import serpapi -from serpapi.http import HTTPClient - - -ROOT = Path(__file__).resolve().parents[1] -PYTHON_BLOCK_RE = re.compile( - r"(?P\s*)?" - r"```python\r?\n(?P.*?)```", - re.DOTALL, -) -PAGE_TIMEOUT = 300 - - -def python_blocks(path): - return [match.groupdict() for match in PYTHON_BLOCK_RE.finditer(path.read_text())] - - -def docs_pages(): - paths = [ROOT / "README.md", *sorted((ROOT / "docs").rglob("*.md"))] - return [path for path in paths if "_build" not in path.parts and python_blocks(path)] - - -def redact(text, api_key=None): - for key in (api_key, os.environ.get("SERPAPI_KEY"), os.environ.get("API_KEY")): - if key: - text = text.replace(key, "[REDACTED]") - return text - - -def validate_response(response, path, params): - if not 200 <= response.status_code < 300: - raise AssertionError(f"HTTP {response.status_code}") - - content_type = response.headers.get("Content-Type", "").lower() - if "json" in content_type: - payload = response.json() - if not isinstance(payload, (dict, list)) or not payload: - raise AssertionError("Expected a nonempty JSON object or list") - if isinstance(payload, dict): - # Google can report an empty final page as Success with an error message. - empty_final_page = ( - path in {"/search", "/search.json"} - and params.get("engine") == "google" - and payload.get("search_metadata", {}).get("status") == "Success" - and payload.get("search_information", {}).get("organic_results_state") == "Fully empty" - and payload.get("error") == "Google hasn't returned any results for this query." - and not payload.get("organic_results") - and not payload.get("serpapi_pagination", {}).get("next") - and int(params.get("start", 0)) > 0 - ) - if payload.get("error") and not empty_final_page: - raise AssertionError(str(payload["error"])) - if payload.get("search_metadata", {}).get("status") == "Error": - raise AssertionError("Search metadata reports an error") - if path == "/image" and not payload.get("image_id"): - raise AssertionError("Image upload did not return image_id") - - if path == "/search" or path.startswith("/searches/"): - result = serpapi.SerpResults.from_http_response(response) - output = params.get("output", "json") - if output in ("md", "html"): - if not isinstance(result, str) or not result.strip(): - raise AssertionError(f"output={output} did not return a nonempty string") - elif not isinstance(result, serpapi.SerpResults): - raise AssertionError("JSON search did not return SerpResults") - - -def install_http_checks(): - original_request = HTTPClient.request - report_path = Path(os.environ["DOCS_EXAMPLE_REPORT_DIR"]) / f"{os.getpid()}.jsonl" - lock = threading.Lock() - - def checked_request(self, method, path, params, **kwargs): - url = urlsplit(path) - query = {key: values[-1] for key, values in parse_qs(url.query).items()} - query.update(params) - record = {"path": url.path, "engine": query.get("engine"), "ok": False} - if not self.timeout and not kwargs.get("timeout"): - kwargs["timeout"] = 30 - try: - response = original_request(self, method, path, params, **kwargs) - validate_response(response, url.path, query) - record["ok"] = True - return response - except Exception as exc: - record["error"] = redact(f"{type(exc).__name__}: {exc}") - raise - finally: - with lock, report_path.open("a") as report: - report.write(json.dumps(record) + "\n") - - HTTPClient.request = checked_request - - -def script_for_page(path): - parts = [ - "import os\nimport serpapi\n" - "from tests.docs_example_support import install_http_checks\n" - "install_http_checks()\n" - 'client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30)\n' - ] - for number, block in enumerate(python_blocks(path), start=1): - if block["skip"]: - if not block["reason"].strip(): - raise RuntimeError(f"Python block {number} needs a docs-test skip reason") - continue - code = block["code"] - for placeholder in ('"secret_api_key"', "'secret_api_key'"): - code = code.replace(placeholder, 'os.environ["SERPAPI_KEY"]') - parts.append(f"# Python block {number}\n{code}") - return "\n\n".join(parts) - - -def run_page(path, workdir, api_key, timeout=PAGE_TIMEOUT): - workdir.mkdir(parents=True, exist_ok=True) - report_dir = workdir / "requests" - report_dir.mkdir() - script = workdir / "example.py" - script.write_text(script_for_page(path)) - shutil.copyfile(ROOT / "assets" / "serpapi-icon.png", workdir / "image.png") - env = os.environ.copy() - env.update({ - "SERPAPI_KEY": api_key, - "API_KEY": api_key, - "DOCS_EXAMPLE_REPORT_DIR": str(report_dir), - "PYTHONPATH": os.pathsep.join([str(ROOT / "tests"), str(ROOT)]), - }) - with subprocess.Popen( - [sys.executable, str(script)], - cwd=workdir, - env=env, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - start_new_session=os.name == "posix", - ) as process: - try: - _, stderr = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - if os.name == "posix": - os.killpg(process.pid, signal.SIGKILL) - else: - process.kill() - process.communicate() - raise RuntimeError(f"Example exceeded the {timeout}-second page limit") from None - - records = [ - json.loads(line) - for report in sorted(report_dir.glob("*.jsonl")) - for line in report.read_text().splitlines() - ] - errors = [record for record in records if not record["ok"]] - if process.returncode or errors: - details = "\n".join( - f"{record['engine'] or record['path']}: {record['error']}" - for record in errors - ) - raise RuntimeError(redact(f"{details}\n{stderr}", api_key).strip()) - if not records: - raise RuntimeError("Example finished without making a SerpApi request") - return records diff --git a/tests/example_search_apple_app_store_test.py b/tests/example_search_apple_app_store_test.py deleted file mode 100644 index 70bb30b..0000000 --- a/tests/example_search_apple_app_store_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: apple_app_store search engine -import pytest -import os -import serpapi - -def test_search_apple_app_store(client): - data = client.search({ - 'engine': 'apple_app_store', - 'term': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_baidu_test.py b/tests/example_search_baidu_test.py deleted file mode 100644 index d5a0d52..0000000 --- a/tests/example_search_baidu_test.py +++ /dev/null @@ -1,13 +0,0 @@ -# Example: baidu search engine -import pytest -import os -import serpapi - - -def test_search_baidu(client): - data = client.search({ - 'engine': 'baidu', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_bing_test.py b/tests/example_search_bing_test.py deleted file mode 100644 index 36a1c79..0000000 --- a/tests/example_search_bing_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: bing search engine -import pytest -import os -import serpapi - -def test_search_bing(client): - data = client.search({ - 'engine': 'bing', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_duckduckgo_test.py b/tests/example_search_duckduckgo_test.py deleted file mode 100644 index 4a198ab..0000000 --- a/tests/example_search_duckduckgo_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: duckduckgo search engine -import pytest -import os -import serpapi - -def test_search_duckduckgo(client): - data = client.search({ - 'engine': 'duckduckgo', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_ebay_test.py b/tests/example_search_ebay_test.py deleted file mode 100644 index fd41037..0000000 --- a/tests/example_search_ebay_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: ebay search engine -import pytest -import os -import serpapi - -def test_search_ebay(client): - data = client.search({ - 'engine': 'ebay', - '_nkw': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_google_autocomplete_test.py b/tests/example_search_google_autocomplete_test.py deleted file mode 100644 index 3de2012..0000000 --- a/tests/example_search_google_autocomplete_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: google_autocomplete search engine -import pytest -import os -import serpapi - -def test_search_google_autocomplete(client): - data = client.search({ - 'engine': 'google_autocomplete', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['suggestions'] diff --git a/tests/example_search_google_images_test.py b/tests/example_search_google_images_test.py deleted file mode 100644 index 338e69b..0000000 --- a/tests/example_search_google_images_test.py +++ /dev/null @@ -1,14 +0,0 @@ -# Example: google_images search engine -import pytest -import os -import serpapi - -def test_search_google_images(client): - data = client.search({ - 'engine': 'google_images', - 'engine': 'google_images', - 'tbm': 'isch', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['images_results'] diff --git a/tests/example_search_google_jobs_test.py b/tests/example_search_google_jobs_test.py deleted file mode 100644 index 809fd65..0000000 --- a/tests/example_search_google_jobs_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: google_jobs search engine -import pytest -import os -import serpapi - -def test_search_google_jobs(client): - data = client.search({ - 'engine': 'google_jobs', - 'q': 'software engineer', - }) - assert data.get('error') is None - assert data['jobs_results'] diff --git a/tests/example_search_google_light_test.py b/tests/example_search_google_light_test.py deleted file mode 100644 index 3213643..0000000 --- a/tests/example_search_google_light_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: google_light search engine -import pytest -import os -import serpapi - -def test_search_google_light(client): - data = client.search({ - 'engine': 'google_light', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_google_local_services_test.py b/tests/example_search_google_local_services_test.py deleted file mode 100644 index 964b013..0000000 --- a/tests/example_search_google_local_services_test.py +++ /dev/null @@ -1,14 +0,0 @@ -# Example: google_local_services search engine -import pytest -import os -import serpapi - -def test_search_google_local_services(client): - - data = client.search({ - 'engine': 'google_local_services', - 'q': 'electrician', - 'data_cid': '6745062158417646970', - }) - assert data.get('error') is None - assert data['local_ads'] diff --git a/tests/example_search_google_maps_test.py b/tests/example_search_google_maps_test.py deleted file mode 100644 index 453b5dc..0000000 --- a/tests/example_search_google_maps_test.py +++ /dev/null @@ -1,14 +0,0 @@ -# Example: google_maps search engine -import pytest -import os -import serpapi - -def test_search_google_maps(client): - data = client.search({ - 'engine': 'google_maps', - 'q': 'pizza', - 'll': '@40.7455096,-74.0083012,15.1z', - 'type': 'search', - }) - assert data.get('error') is None - assert data['local_results'] diff --git a/tests/example_search_google_play_test.py b/tests/example_search_google_play_test.py deleted file mode 100644 index bd44bef..0000000 --- a/tests/example_search_google_play_test.py +++ /dev/null @@ -1,14 +0,0 @@ -# Example: google_play search engine -import pytest -import os -import serpapi - -def test_search_google_play(client): - data = client.search({ - 'engine': 'google_play', - 'q': 'kite', - 'store': 'apps', - 'max_results': '2', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_google_scholar_test.py b/tests/example_search_google_scholar_test.py deleted file mode 100644 index 61ff7e0..0000000 --- a/tests/example_search_google_scholar_test.py +++ /dev/null @@ -1,13 +0,0 @@ -# Example: google_scholar search engine -import pytest -import os -import serpapi - -def test_search_google_scholar(client): - - data = client.search({ - 'engine': 'google_scholar', - 'q': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_google_test.py b/tests/example_search_google_test.py deleted file mode 100644 index 650bcff..0000000 --- a/tests/example_search_google_test.py +++ /dev/null @@ -1,14 +0,0 @@ -# Example: google search engine -import pytest -import os -import serpapi - -def test_search_google(client): - - data = client.search({ - 'engine': 'google', - 'q': 'coffee', - 'engine': 'google', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_home_depot_test.py b/tests/example_search_home_depot_test.py deleted file mode 100644 index 5d6e2a5..0000000 --- a/tests/example_search_home_depot_test.py +++ /dev/null @@ -1,13 +0,0 @@ -# Example: home_depot search engine -import pytest -import os -import serpapi - -def test_search_home_depot(client): - - data = client.search({ - 'engine': 'home_depot', - 'q': 'chair', - }) - assert data.get('error') is None - assert data['products'] diff --git a/tests/example_search_naver_test.py b/tests/example_search_naver_test.py deleted file mode 100644 index fa2bb26..0000000 --- a/tests/example_search_naver_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: naver search engine -import pytest -import os -import serpapi - -def test_search_naver(client): - data = client.search({ - 'engine': 'naver', - 'query': 'coffee', - }) - assert data.get('error') is None - assert data['ads_results'] diff --git a/tests/example_search_walmart_test.py b/tests/example_search_walmart_test.py deleted file mode 100644 index 24248fa..0000000 --- a/tests/example_search_walmart_test.py +++ /dev/null @@ -1,13 +0,0 @@ -# Example: walmart search engine -import pytest -import os -import serpapi - -def test_search_walmart(client): - data = client.search({ - 'engine': 'walmart', - 'query': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] - diff --git a/tests/example_search_yahoo_test.py b/tests/example_search_yahoo_test.py deleted file mode 100644 index 0a5af69..0000000 --- a/tests/example_search_yahoo_test.py +++ /dev/null @@ -1,12 +0,0 @@ -# Example: yahoo search engine -import pytest -import os -import serpapi - -def test_search_yahoo(client): - data = client.search({ - 'engine': 'yahoo', - 'p': 'coffee', - }) - assert data.get('error') is None - assert data['organic_results'] diff --git a/tests/example_search_youtube_test.py b/tests/example_search_youtube_test.py deleted file mode 100644 index 00c3da6..0000000 --- a/tests/example_search_youtube_test.py +++ /dev/null @@ -1,13 +0,0 @@ -# Example: youtube search engine -import pytest -import os -import serpapi - -def test_search_youtube(client): - - data = client.search({ - 'engine': 'youtube', - 'search_query': 'coffee', - }) - assert data.get('error') is None - assert data['video_results'] diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..f3edc54 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,281 @@ +""" +Test suite for SerpApi Client. +""" + +import asyncio +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from serpapi import Client, SerpApiError + + +class TestClient: + """Test cases for SerpApi Client.""" + + def test_client_initialization_with_api_key(self): + """Test client initialization with API key.""" + client = Client(api_key="test_key", engine="google") + assert client.api_key == "test_key" + assert client.engine == "google" + assert client.persistent is True + assert client.timeout == 120 + + def test_client_initialization_with_env_var(self): + """Test client initialization with environment variable.""" + with patch.dict(os.environ, {"SERPAPI_KEY": "env_key"}): + client = Client(engine="bing") + assert client.api_key == "env_key" + assert client.engine == "bing" + + def test_client_initialization_no_api_key(self): + """Test client initialization without API key raises error.""" + with patch.dict(os.environ, {}, clear=True): + with pytest.raises(SerpApiError, match="API key is required"): + Client() + + def test_client_initialization_custom_params(self): + """Test client initialization with custom parameters.""" + client = Client( + api_key="test_key", + engine="yahoo", + persistent=False, + timeout=60, + custom_param="value", + ) + assert client.api_key == "test_key" + assert client.engine == "yahoo" + assert client.persistent is False + assert client.timeout == 60 + assert client._params["custom_param"] == "value" + + def test_merge_params(self): + """Test parameter merging.""" + client = Client(api_key="test_key", engine="google", param1="value1") + + # Test with valid params + merged = client._merge_params({"param2": "value2", "q": "coffee"}) + expected = { + "api_key": "test_key", + "engine": "google", + "source": "serpapi-python:1.0.1", + "param1": "value1", + "param2": "value2", + "q": "coffee", + } + assert merged == expected + + # Test with invalid params + with pytest.raises(SerpApiError, match="params must be dict"): + client._merge_params("invalid") + + @pytest.mark.asyncio + async def test_search_success(self): + """Test successful search request.""" + client = Client(api_key="test_key") + + mock_response = { + "search_metadata": {"id": "test_id", "status": "Success"}, + "organic_results": [ + {"title": "Test Result", "link": "https://example.com"} + ], + } + + with patch.object( + client, "_make_request", return_value=mock_response + ) as mock_request: + result = await client.search({"q": "coffee"}) + + mock_request.assert_called_once_with("/search", {"q": "coffee"}, "json") + assert result == mock_response + + @pytest.mark.asyncio + async def test_html_success(self): + """Test successful HTML request.""" + client = Client(api_key="test_key") + + mock_html = "Test HTML" + + with patch.object( + client, "_make_request", return_value=mock_html + ) as mock_request: + result = await client.html({"q": "coffee"}) + + mock_request.assert_called_once_with("/search", {"q": "coffee"}, "html") + assert result == mock_html + + @pytest.mark.asyncio + async def test_location_success(self): + """Test successful location request.""" + client = Client(api_key="test_key") + + mock_locations = [{"id": "1", "name": "Austin, TX", "country_code": "US"}] + + with patch.object( + client, "_make_request", return_value=mock_locations + ) as mock_request: + result = await client.location({"q": "Austin", "limit": 3}) + + mock_request.assert_called_once_with( + "/locations.json", {"q": "Austin", "limit": 3}, "json" + ) + assert result == mock_locations + + @pytest.mark.asyncio + async def test_search_archive_success(self): + """Test successful search archive request.""" + client = Client(api_key="test_key") + + mock_archive = {"search_metadata": {"id": "test_id"}, "organic_results": []} + + with patch.object( + client, "_make_request", return_value=mock_archive + ) as mock_request: + result = await client.search_archive("test_id", "json") + + mock_request.assert_called_once_with("/searches/test_id.json", {}, "json") + assert result == mock_archive + + @pytest.mark.asyncio + async def test_search_archive_invalid_format(self): + """Test search archive with invalid format.""" + client = Client(api_key="test_key") + + with pytest.raises(SerpApiError, match="format_type must be json or html"): + await client.search_archive("test_id", "invalid") + + @pytest.mark.asyncio + async def test_account_success(self): + """Test successful account request.""" + client = Client(api_key="test_key") + + mock_account = { + "account_id": "123456", + "account_email": "test@example.com", + "plan_name": "Free Plan", + } + + with patch.object( + client, "_make_request", return_value=mock_account + ) as mock_request: + result = await client.account() + + mock_request.assert_called_once_with("/account", {}, "json") + assert result == mock_account + + @pytest.mark.asyncio + async def test_account_with_api_key(self): + """Test account request with specific API key.""" + client = Client(api_key="test_key") + + mock_account = {"account_id": "123456"} + + with patch.object( + client, "_make_request", return_value=mock_account + ) as mock_request: + result = await client.account("custom_key") + + mock_request.assert_called_once_with( + "/account", {"api_key": "custom_key"}, "json" + ) + assert result == mock_account + + @pytest.mark.asyncio + async def test_make_request_json_success(self): + """Test successful JSON request.""" + client = Client(api_key="test_key") + + # Test the public API methods instead of internal _make_request + with patch.object(client, "_make_request") as mock_request: + mock_request.return_value = {"result": "success"} + + result = await client.search({"q": "test"}) + + assert result == {"result": "success"} + mock_request.assert_called_once() + + @pytest.mark.asyncio + async def test_make_request_json_error_response(self): + """Test JSON request with error response.""" + client = Client(api_key="test_key") + + # Test error handling through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.side_effect = SerpApiError("SerpApi error: API Error") + + with pytest.raises(SerpApiError, match="SerpApi error: API Error"): + await client.search({"q": "test"}) + + @pytest.mark.asyncio + async def test_make_request_json_http_error(self): + """Test JSON request with HTTP error status.""" + client = Client(api_key="test_key") + + # Test HTTP error handling through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.side_effect = SerpApiError("SerpApi error: Bad Request") + + with pytest.raises(SerpApiError, match="SerpApi error: Bad Request"): + await client.search({"q": "test"}) + + @pytest.mark.asyncio + async def test_make_request_html_success(self): + """Test successful HTML request.""" + client = Client(api_key="test_key") + + # Test HTML request through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.return_value = "Test" + + result = await client.html({"q": "test"}) + + assert result == "Test" + mock_request.assert_called_once() + + @pytest.mark.asyncio + async def test_make_request_invalid_format(self): + """Test request with invalid response format.""" + client = Client(api_key="test_key") + + # Test invalid format through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.side_effect = SerpApiError( + "Unsupported response format: invalid" + ) + + with pytest.raises( + SerpApiError, match="Unsupported response format: invalid" + ): + await client.search({"q": "test"}) + + @pytest.mark.asyncio + async def test_context_manager(self): + """Test async context manager.""" + with patch.object(Client, "close") as mock_close: + async with Client(api_key="test_key") as client: + assert isinstance(client, Client) + + mock_close.assert_called_once() + + @pytest.mark.asyncio + async def test_close_session(self): + """Test closing session.""" + client = Client(api_key="test_key") + + mock_session = AsyncMock() + mock_session.closed = False + client._session = mock_session + + await client.close() + + mock_session.close.assert_called_once() + + def test_properties(self): + """Test client properties.""" + client = Client(api_key="test_key", engine="google", timeout=60) + + assert client.timeout == 60 + assert client.persistent is True + assert client.engine == "google" + assert client.api_key == "test_key" diff --git a/tests/test_client_integration.py b/tests/test_client_integration.py new file mode 100644 index 0000000..36ac9f8 --- /dev/null +++ b/tests/test_client_integration.py @@ -0,0 +1,134 @@ +""" +Integration-style test suite for SerpApi Client. +Focuses on covering HTTP request logic and session management. +""" + +import asyncio +import json +from unittest.mock import AsyncMock, patch, MagicMock + +import pytest +import aiohttp +from serpapi import Client, SerpApiError + +class TestClientIntegration: + """Tests that exercise internal _make_request and session logic.""" + + @pytest.mark.asyncio + async def test_make_request_json_error_parsing(self): + """Covers error parsing from JSON response when status != 200.""" + client = Client(api_key="test_key") + + mock_response = AsyncMock() + mock_response.status = 403 + mock_response.json.return_value = {"error": "Invalid API key"} + + # We need to mock the context manager returned by session.get + mock_get = MagicMock() + mock_get.__aenter__.return_value = mock_response + + with patch("aiohttp.ClientSession.get", return_value=mock_get): + with pytest.raises(SerpApiError, match="SerpApi error: Invalid API key"): + await client.search({"q": "coffee"}) + + @pytest.mark.asyncio + async def test_make_request_non_json_error(self): + """Covers error handling when response is not JSON and status != 200.""" + client = Client(api_key="test_key") + + mock_response = AsyncMock() + mock_response.status = 500 + mock_response.json.side_effect = json.JSONDecodeError("Expecting value", "", 0) + mock_response.text.return_value = "Internal Server Error" + + mock_get = MagicMock() + mock_get.__aenter__.return_value = mock_response + + with patch("aiohttp.ClientSession.get", return_value=mock_get): + with pytest.raises(SerpApiError, match="HTTP request failed with error: Internal Server Error"): + await client.search({"q": "coffee"}) + + @pytest.mark.asyncio + async def test_make_request_invalid_json_success_status(self): + """Covers JSONDecodeError when status is 200 but body is invalid JSON.""" + client = Client(api_key="test_key") + + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json.side_effect = json.JSONDecodeError("Expecting value", "", 0) + mock_response.text.return_value = "Not JSON" + + mock_get = MagicMock() + mock_get.__aenter__.return_value = mock_response + + with patch("aiohttp.ClientSession.get", return_value=mock_get): + with pytest.raises(SerpApiError, match="Invalid JSON response: Not JSON"): + await client.search({"q": "coffee"}) + + @pytest.mark.asyncio + async def test_make_request_unsupported_format(self): + """Covers the 'else' branch for unsupported response format.""" + client = Client(api_key="test_key") + + mock_response = AsyncMock() + mock_response.status = 200 + + mock_get = MagicMock() + mock_get.__aenter__.return_value = mock_response + + with patch("aiohttp.ClientSession.get", return_value=mock_get): + with pytest.raises(SerpApiError, match="Unsupported response format: xml"): + await client._make_request("/search", {"q": "test"}, response_format="xml") + + @pytest.mark.asyncio + async def test_session_creation_and_persistence(self): + """Covers lines 108-122: Session creation, lock, and reuse.""" + client = Client(api_key="test_key", persistent=True) + + # First call creates session + session1 = await client._get_session() + assert isinstance(session1, aiohttp.ClientSession) + assert session1.closed is False + + # Second call returns same session + session2 = await client._get_session() + assert session1 is session2 + + await client.close() + assert session1.closed is True + + # Call after close creates NEW session + session3 = await client._get_session() + assert session3 is not session1 + assert session3.closed is False + await client.close() + + @pytest.mark.asyncio + async def test_non_persistent_session(self): + """Covers session behavior when persistent=False.""" + client = Client(api_key="test_key", persistent=False) + + session1 = await client._get_session() + session2 = await client._get_session() + + # Even if not persistent, _get_session might return same if not closed + # but the key is that we test the branch + assert session1 is not None + await client.close() + + @pytest.mark.asyncio + async def test_client_del_cleanup(self): + """Covers __del__ cleanup logic.""" + client = Client(api_key="test_key") + session = await client._get_session() + assert session.closed is False + + # Manually call __del__ since we can't easily trigger GC reliably in test + with patch.object(asyncio, "get_event_loop") as mock_loop: + mock_loop_instance = MagicMock() + mock_loop_instance.is_running.return_value = True + mock_loop.return_value = mock_loop_instance + + client.__del__() + mock_loop_instance.create_task.assert_called_once() + diff --git a/tests/test_docs_example_runner.py b/tests/test_docs_example_runner.py deleted file mode 100644 index c0d5f8c..0000000 --- a/tests/test_docs_example_runner.py +++ /dev/null @@ -1,158 +0,0 @@ -import json -import os -import subprocess -import sys -from textwrap import dedent - -import pytest -import requests - -from tests.docs_example_support import ROOT, run_page, validate_response - - -MOCK_REQUEST = """ -import json -import requests - -def fake_request(self, **kwargs): - response = requests.Response() - response.status_code = 200 - response.headers['Content-Type'] = 'application/json' - response._content = json.dumps({'organic_results': [{'title': 'Coffee'}]}).encode() - return response - -requests.Session.request = fake_request -""" - - -def markdown_page(tmp_path, code): - path = tmp_path / "page.md" - path.write_text("```python\n" + dedent(code) + "\n```\n") - return path - - -def test_doc_runner_executes_main_guard_and_spawned_workers(tmp_path): - code = MOCK_REQUEST + """ -from concurrent.futures import ProcessPoolExecutor -import multiprocessing - -def search_worker(query): - results = client.search(q=query) - return results['organic_results'][0]['title'] - -if __name__ == '__main__': - with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context('spawn')) as pool: - assert list(pool.map(search_worker, ['one', 'two'])) == ['Coffee', 'Coffee'] -""" - page = markdown_page(tmp_path, code) - records = run_page(page, tmp_path / "run", "test-key") - assert len(records) == 2 - assert all(record["ok"] for record in records) - - -def test_doc_runner_preserves_block_state_and_skips_marked_examples(tmp_path): - page = markdown_page(tmp_path, MOCK_REQUEST + "\nquery = 'coffee'\n") - with page.open("a") as source: - source.write(""" - -```python -raise RuntimeError('This block must not run') -``` -```python -from pathlib import Path -assert Path('image.png').read_bytes().startswith(b'\\x89PNG') -client = serpapi.Client(api_key='secret_api_key') -assert client.api_key == os.environ['SERPAPI_KEY'] -results = client.search(q=query) -assert results['organic_results'][0]['title'] == 'Coffee' -``` -""") - assert len(run_page(page, tmp_path / "run", "test-key")) == 1 - assert "test-key" not in (tmp_path / "run" / "example.py").read_text() - - -def test_doc_runner_fails_on_caught_api_errors_and_redacts_key(tmp_path): - code = MOCK_REQUEST.replace( - "{'organic_results': [{'title': 'Coffee'}]}", - "{'error': os.environ['SERPAPI_KEY']}", - ) + """ -try: - client.search(q='coffee') -except AssertionError: - pass -""" - page = markdown_page(tmp_path, code) - with pytest.raises(RuntimeError) as failure: - run_page(page, tmp_path / "run", "private-test-key") - assert "[REDACTED]" in str(failure.value) - assert "private-test-key" not in str(failure.value) - for report in (tmp_path / "run" / "requests").glob("*.jsonl"): - assert "private-test-key" not in report.read_text() - - -def test_doc_runner_rejects_examples_without_requests(tmp_path): - page = markdown_page(tmp_path, "if __name__ == 'docs_examples':\n client.search(q='coffee')") - with pytest.raises(RuntimeError, match="without making a SerpApi request"): - run_page(page, tmp_path / "run", "test-key") - - -def test_doc_runner_stops_a_stalled_example(tmp_path): - page = markdown_page(tmp_path, "import time\ntime.sleep(60)") - with pytest.raises(RuntimeError, match="page limit"): - run_page(page, tmp_path / "run", "test-key", timeout=1) - - -@pytest.mark.parametrize("output", ["md", "html"]) -def test_doc_response_check_rejects_json_for_text_output(output): - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = "application/json" - response._content = b'{"organic_results": [{"title": "Coffee"}]}' - with pytest.raises(AssertionError, match="nonempty string"): - validate_response(response, "/search", {"output": output}) - - -@pytest.mark.parametrize( - "start, overrides, accepted", - [ - pytest.param(70, {}, True, id="empty-final-page"), - pytest.param(0, {}, False, id="empty-first-page"), - pytest.param(70, {"search_metadata": {"status": "Error"}}, False, - id="failed-search"), - pytest.param(70, {"error": "Failed to fetch results"}, False, - id="different-error"), - pytest.param(70, {"serpapi_pagination": {"next": "https://serpapi.com/search?start=80"}}, - False, id="not-final-page"), - ], -) -def test_doc_response_check_handles_empty_google_pagination(start, overrides, accepted): - payload = { - "search_metadata": {"status": "Success"}, - "search_information": {"organic_results_state": "Fully empty"}, - "error": "Google hasn't returned any results for this query.", - } - payload.update(overrides) - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = "application/json" - response._content = json.dumps(payload).encode() - params = {"engine": "google", "start": str(start)} - - if accepted: - validate_response(response, "/search.json", params) - else: - with pytest.raises(AssertionError): - validate_response(response, "/search.json", params) - - -def test_doc_gate_fails_without_a_key(): - env = os.environ.copy() - env.pop("SERPAPI_KEY", None) - env.pop("API_KEY", None) - result = subprocess.run( - [sys.executable, "-m", "pytest", "tests/test_docs_examples.py", - "--require-docs-key", "--collect-only", "-q"], - cwd=ROOT, env=env, capture_output=True, text=True, - ) - assert result.returncode != 0 - assert "Live docs checks require SERPAPI_KEY or API_KEY" in result.stderr diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py deleted file mode 100644 index 3d7494f..0000000 --- a/tests/test_docs_examples.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -import pytest - -from tests.docs_example_support import ROOT, docs_pages, python_blocks, run_page - - -PAGES = docs_pages() - - -@pytest.mark.parametrize("path", PAGES, ids=lambda path: path.relative_to(ROOT).as_posix()) -def test_documentation_python_blocks_compile(path): - for number, block in enumerate(python_blocks(path), start=1): - compile(block["code"], f"{path}::python-block-{number}", "exec") - - -@pytest.mark.parametrize("path", PAGES, ids=lambda path: path.relative_to(ROOT).as_posix()) -def test_documentation_examples_live(path, tmp_path): - api_key = os.environ.get("SERPAPI_KEY") or os.environ.get("API_KEY") - if not api_key: - pytest.skip("Set SERPAPI_KEY or API_KEY to run live documentation examples") - - try: - run_page(path, tmp_path, api_key) - except RuntimeError as exc: - pytest.fail(str(exc), pytrace=False) diff --git a/tests/test_docs_publishing.py b/tests/test_docs_publishing.py deleted file mode 100644 index 7c34675..0000000 --- a/tests/test_docs_publishing.py +++ /dev/null @@ -1,315 +0,0 @@ -import io -import json -from collections import deque -from pathlib import Path -from shlex import quote -import subprocess -from types import SimpleNamespace -from urllib.error import HTTPError - -import pytest - -from scripts import check_docs_revision as guard -from scripts import publish_docs as publisher - - -COMMIT = "a" * 40 -TOKEN = "private-test-token" - - -def ci_environment(**context_changes): - context = {"commit": COMMIT, "versions": ["latest"], "expires_at": 200} - context.update(context_changes) - return { - "READTHEDOCS_VERSION": "latest", - "READTHEDOCS_VERSION_NAME": "latest", - "READTHEDOCS_VERSION_TYPE": "branch", - guard.CONTEXT_VARIABLE: json.dumps(context), - } - - -@pytest.mark.parametrize("shell_quoted", [False, True]) -def test_revision_guard_accepts_only_the_tested_commit(shell_quoted): - env = ci_environment() - if shell_quoted: - env[guard.CONTEXT_VARIABLE] = quote(env[guard.CONTEXT_VARIABLE]) - guard.check_revision(env, COMMIT, now=100) - with pytest.raises(RuntimeError, match="different commit"): - guard.check_revision(env, "b" * 40, now=100) - - -@pytest.mark.parametrize("context", [ - None, "not json", "null", "[]", "{}", "'unterminated", "'{}' '{}'", quote("not json"), -]) -def test_revision_guard_rejects_missing_or_malformed_context(context): - env = ci_environment() - if context is None: - env.pop(guard.CONTEXT_VARIABLE) - else: - env[guard.CONTEXT_VARIABLE] = context - with pytest.raises(RuntimeError, match="No valid CI revision"): - guard.check_revision(env, COMMIT, now=100) - - -def test_revision_guard_rejects_expired_and_other_versions(): - with pytest.raises(RuntimeError, match="expired"): - guard.check_revision(ci_environment(expires_at=100), COMMIT, now=100) - with pytest.raises(RuntimeError, match="did not request"): - guard.check_revision(ci_environment(versions=["v1.2.3", "stable"]), COMMIT, now=100) - - -def test_revision_guard_uses_tag_name_instead_of_normalized_slug(): - env = ci_environment(versions=["v1.2.3", "stable"]) - env.update(READTHEDOCS_VERSION="v123", READTHEDOCS_VERSION_NAME="v1.2.3", - READTHEDOCS_VERSION_TYPE="tag") - guard.check_revision(env, COMMIT, now=100) - env["READTHEDOCS_VERSION_NAME"] = "stable" - guard.check_revision(env, COMMIT, now=100) - env["READTHEDOCS_VERSION"] = "stable" - env["READTHEDOCS_VERSION_NAME"] = "v9.0.0" - with pytest.raises(RuntimeError, match="did not request"): - guard.check_revision(env, COMMIT, now=100) - - -def test_revision_guard_falls_back_to_slug_when_name_is_missing(): - env = ci_environment() - env.pop("READTHEDOCS_VERSION_NAME") - guard.check_revision(env, COMMIT, now=100) - - -def test_pr_preview_needs_no_ci_context(): - guard.check_revision({"READTHEDOCS_VERSION_TYPE": "external"}, COMMIT) - - -def page(*items, next=None): - return {"results": list(items), "next": next} - - -def build(build_id=10, **changes): - result = { - "id": build_id, "version": "latest", "commit": COMMIT, - "state": {"code": "finished"}, "success": True, - } - result.update(changes) - return result - - -class HTTPResponses: - # Route requests by endpoint; successive responses model asynchronous changes. - def __init__(self): - self.routes = {} - self.requests = [] - - def respond(self, method, path, *responses): - self.routes[method, path] = deque(responses) - - def open(self, request, timeout): - assert request.get_header("Authorization") == f"Token {TOKEN}" - assert request.get_header("Content-type") == "application/json" - assert TOKEN not in request.full_url - assert timeout > 0 - base = f"{publisher.API_ROOT}projects/{publisher.PROJECT}/" - assert request.full_url.startswith(base) - path = request.full_url[len(base):] - data = json.loads(request.data) if request.data is not None else None - self.requests.append((request.get_method(), path, data)) - responses = self.routes[request.get_method(), path] - response = responses.popleft() if len(responses) > 1 else responses[0] - if isinstance(response, Exception): - raise response - if not isinstance(response, bytes): - response = b"" if response is None else json.dumps(response).encode() - return io.BytesIO(response) - - def sent(self, method, path): - return [data for verb, endpoint, data in self.requests - if (verb, endpoint) == (method, path)] - - -@pytest.fixture -def clock(monkeypatch): - clock = SimpleNamespace(elapsed=0) - - def sleep(seconds): - clock.elapsed += seconds - - monkeypatch.setattr(publisher, "time", SimpleNamespace( - time=lambda: 100, monotonic=lambda: clock.elapsed, sleep=sleep, - )) - return clock - - -@pytest.fixture -def rtd(monkeypatch, clock): - rtd = HTTPResponses() - rtd.respond("GET", "", {"default_branch": "master"}) - rtd.respond("GET", "builds/?running=true", page()) - rtd.respond("GET", "environmentvariables/", page({"name": "UNRELATED", "pk": 1})) - rtd.respond("GET", "builds/?limit=1", page(build(9))) - rtd.respond("POST", "environmentvariables/", {"pk": 2}) - rtd.respond("POST", "sync-versions/", {}) - rtd.respond("GET", "versions/latest/", {"slug": "latest", "active": True}) - rtd.respond("GET", "builds/", page(build(9))) - rtd.respond("POST", "versions/latest/builds/", {"build": build()}) - rtd.respond("GET", "builds/10/", build()) - rtd.respond("DELETE", "environmentvariables/2/", None) - monkeypatch.setattr(publisher, "build_opener", lambda *handlers: SimpleNamespace(open=rtd.open)) - rtd.api = publisher.ReadTheDocs(TOKEN) - return rtd - - -@pytest.fixture -def github_run(monkeypatch, rtd): - monkeypatch.chdir(Path(__file__).resolve().parents[1]) - commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() - for name, value in { - "RTD_API_TOKEN": TOKEN, "GITHUB_EVENT_NAME": "push", - "GITHUB_SHA": commit, "GITHUB_REF": "refs/heads/master", - }.items(): - monkeypatch.setenv(name, value) - rtd.respond("GET", "builds/10/", build(commit=commit)) - return commit - - -@pytest.mark.parametrize("event", ["push", "workflow_dispatch"]) -def test_main_publishes_the_checkout_with_scoped_context_and_cleanup(monkeypatch, rtd, github_run, event): - monkeypatch.setenv("GITHUB_EVENT_NAME", event) - publisher.main() - creation, = rtd.sent("POST", "environmentvariables/") - assert creation["public"] is False - assert creation["name"] == guard.CONTEXT_VARIABLE - context = json.loads(creation["value"]) - assert context["commit"] == github_run - assert context["versions"] == ["latest"] - assert context["expires_at"] > publisher.time.time() - assert rtd.sent("DELETE", "environmentvariables/2/") == [None] - actions = [(method, path) for method, path, _ in rtd.requests] - assert (actions.index(("POST", "environmentvariables/")) - < actions.index(("POST", "sync-versions/")) - < actions.index(("POST", "versions/latest/builds/")) - < actions.index(("DELETE", "environmentvariables/2/"))) - - -@pytest.mark.parametrize("name, value, message", [ - ("RTD_API_TOKEN", None, "Add RTD_API_TOKEN"), - ("GITHUB_EVENT_NAME", "pull_request", "PR runs cannot publish"), - ("GITHUB_SHA", "0" * 40, "checkout does not match"), -]) -def test_main_rejects_invalid_ci_environment_before_contacting_rtd(monkeypatch, rtd, github_run, name, value, message): - if value is None: - monkeypatch.delenv(name) - else: - monkeypatch.setenv(name, value) - with pytest.raises(RuntimeError, match=message): - publisher.main() - assert not rtd.requests - - -@pytest.mark.parametrize("stable_ref", ["v1.2.3", "v2.0.0", None]) -def test_release_discovers_and_activates_tag_then_updates_both_aliases_for_stable(rtd, clock, stable_ref): - tag = {"slug": "v123", "active": False} - rtd.respond("GET", "versions/?type=tag&verbose_name=v1.2.3", page(), page(tag)) - rtd.respond("PATCH", "versions/v123/", None) - rtd.respond("GET", "builds/", page(build(9)), page(build(9)), - page(build(version="v123"))) - rtd.respond("GET", "builds/10/", build(state={"code": "building"}, success=None), - build(version="v123")) - stable = ({"slug": "stable", "active": True, "ref": stable_ref} - if stable_ref else HTTPError(rtd.api.base + "versions/stable/", 404, "Not found", {}, None)) - rtd.respond("GET", "versions/stable/", stable) - rtd.respond("POST", "versions/stable/builds/", {"build": build(11, version="stable")}) - rtd.respond("GET", "builds/11/", build(11, version="stable")) - rtd.respond("POST", "versions/latest/builds/", {"build": build(12)}) - rtd.respond("GET", "builds/12/", build(12)) - - publisher.publish(rtd.api, "refs/tags/v1.2.3", COMMIT) - - assert clock.elapsed > 0 - assert rtd.sent("PATCH", "versions/v123/") == [{"active": True, "hidden": False}] - assert not rtd.sent("POST", "versions/v123/builds/") - assert len(rtd.sent("POST", "versions/stable/builds/")) == (stable_ref == "v1.2.3") - assert len(rtd.sent("POST", "versions/latest/builds/")) == (stable_ref == "v1.2.3") - creation, = rtd.sent("POST", "environmentvariables/") - assert json.loads(creation["value"])["versions"] == ["v1.2.3", "stable", "latest"] - assert rtd.sent("DELETE", "environmentvariables/2/") == [None] - - -def test_release_rejects_latest_if_master_moved_past_tested_tag(rtd): - rtd.respond("GET", "versions/?type=tag&verbose_name=v1.2.3", - page({"slug": "v123", "active": True})) - rtd.respond("POST", "versions/v123/builds/", {"build": build(10, version="v123")}) - rtd.respond("GET", "builds/10/", build(10, version="v123")) - rtd.respond("GET", "versions/stable/", {"slug": "stable", "active": True, "ref": "v1.2.3"}) - rtd.respond("POST", "versions/stable/builds/", {"build": build(11, version="stable")}) - rtd.respond("GET", "builds/11/", build(11, version="stable")) - rtd.respond("POST", "versions/latest/builds/", {"build": build(12)}) - rtd.respond("GET", "builds/12/", build(12, commit="b" * 40)) - - with pytest.raises(RuntimeError, match="different commit"): - publisher.publish(rtd.api, "refs/tags/v1.2.3", COMMIT) - - assert rtd.sent("DELETE", "environmentvariables/2/") == [None] - - -def test_publisher_reuses_matching_build_started_by_sync(rtd): - rtd.respond("GET", "builds/", page(build(11, version="v123"), build(10, version="stable"))) - rtd.respond("GET", "builds/10/", build(version="stable")) - result = publisher.publish_version(rtd.api, {"slug": "stable", "active": True}, COMMIT, after=9) - assert result["id"] == 10 - assert all(method == "GET" for method, _, _ in rtd.requests) - - -@pytest.mark.parametrize("outcome, message", [ - (build(success=False), "failed or was cancelled"), - (build(state={"code": "cancelled"}, success=None), "failed or was cancelled"), - (build(commit="b" * 40), "different commit"), - (build(state={"code": "building"}, success=None), "Timed out waiting"), -]) -def test_failed_or_stalled_publication_removes_context(rtd, outcome, message): - rtd.respond("GET", "builds/10/", outcome) - with pytest.raises(RuntimeError, match=message): - publisher.publish(rtd.api, "refs/heads/master", COMMIT) - assert len(rtd.sent("POST", "environmentvariables/")) == 1 - assert rtd.sent("DELETE", "environmentvariables/2/") == [None] - - -def test_publisher_waits_for_existing_builds_before_changing_context(rtd): - rtd.respond("GET", "builds/?running=true", page(build()), page()) - publisher.publish(rtd.api, "refs/heads/master", COMMIT) - actions = [(method, path) for method, path, _ in rtd.requests] - polls = [index for index, action in enumerate(actions) if action == ("GET", "builds/?running=true")] - assert len(polls) >= 2 - assert max(polls) < actions.index(("POST", "environmentvariables/")) - - -def test_context_cleanup_follows_pagination_and_preserves_other_variables(rtd): - next_url = rtd.api.base + "environmentvariables/?offset=1" - rtd.respond("GET", "environmentvariables/", page({"name": "UNRELATED", "pk": 1}, next=next_url)) - rtd.respond("GET", "environmentvariables/?offset=1", page({"name": guard.CONTEXT_VARIABLE, "pk": 2})) - publisher.clear_context(rtd.api) - assert [(path, data) for method, path, data in rtd.requests if method == "DELETE"] == [ - ("environmentvariables/2/", None), - ] - - -def test_api_keeps_token_out_of_errors_and_rejects_external_urls(rtd): - rtd.respond("GET", "", HTTPError(rtd.api.base, 403, TOKEN, {}, io.BytesIO(TOKEN.encode()))) - with pytest.raises(publisher.APIError) as failure: - rtd.api.request("GET", "") - assert TOKEN not in str(failure.value) - assert failure.value.status == 403 - with pytest.raises(RuntimeError, match="outside"): - rtd.api.request("GET", "https://example.com/") - - -def test_api_reports_invalid_json_response(rtd): - rtd.respond("GET", "", b"Service unavailable") - with pytest.raises(RuntimeError, match="invalid API response"): - rtd.api.request("GET", "") - - -@pytest.mark.parametrize("ref", ["refs/heads/feature", "refs/pull/1/merge", "refs/tags/test"]) -def test_publisher_rejects_refs_outside_release_policy(rtd, ref): - with pytest.raises(RuntimeError, match="Only master"): - publisher.publish(rtd.api, ref, COMMIT) - assert not rtd.requests diff --git a/tests/test_error.py b/tests/test_error.py new file mode 100644 index 0000000..b81724a --- /dev/null +++ b/tests/test_error.py @@ -0,0 +1,37 @@ +""" +Test suite for SerpApi Error handling. +""" + +import pytest + +from serpapi import SerpApiError + + +class TestSerpApiError: + """Test cases for SerpApiError exception.""" + + def test_serpapi_error_inheritance(self): + """Test that SerpApiError inherits from Exception.""" + error = SerpApiError("Test error message") + assert isinstance(error, Exception) + assert str(error) == "Test error message" + + def test_serpapi_error_with_message(self): + """Test SerpApiError with custom message.""" + message = "API request failed" + error = SerpApiError(message) + assert str(error) == message + + def test_serpapi_error_raise(self): + """Test raising SerpApiError.""" + with pytest.raises(SerpApiError, match="Test error"): + raise SerpApiError("Test error") + + def test_serpapi_error_catch(self): + """Test catching SerpApiError.""" + try: + raise SerpApiError("Caught error") + except SerpApiError as e: + assert str(e) == "Caught error" + except Exception: + pytest.fail("SerpApiError should be caught") diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py deleted file mode 100644 index 02c0793..0000000 --- a/tests/test_exceptions.py +++ /dev/null @@ -1,33 +0,0 @@ -from unittest.mock import Mock -import pytest -import requests -import serpapi - - -def test_http_error(): - """Ensure that an HTTPError has the correct status code and error.""" - mock_response = Mock() - mock_response.status_code = 401 - mock_response.json.return_value = { "error": "Invalid API key" } - - requests_error = requests.exceptions.HTTPError(response=mock_response, request=Mock()) - http_error = serpapi.HTTPError(requests_error) - - assert http_error.status_code == 401 - assert http_error.error == "Invalid API key" - assert http_error.response == mock_response - - -@pytest.mark.parametrize( - "cause, expected", - [ - (requests.exceptions.Timeout, serpapi.TimeoutError), - (requests.exceptions.ConnectionError, serpapi.HTTPConnectionError), - ], -) -def test_transport_errors_are_translated(monkeypatch, cause, expected): - client = serpapi.Client(api_key="test-key") - monkeypatch.setattr(client.session, "request", Mock(side_effect=cause("Request failed"))) - - with pytest.raises(expected): - client.search(engine="google", q="coffee") diff --git a/tests/test_extract.py b/tests/test_extract.py deleted file mode 100644 index 3c7121b..0000000 --- a/tests/test_extract.py +++ /dev/null @@ -1,155 +0,0 @@ -import os -from unittest.mock import Mock -from urllib.parse import urlparse - -import pytest -import requests - -import serpapi - - -@pytest.fixture -def extract_request(monkeypatch): - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = "application/json" - response._content = b'{"markdown": "Example content."}' - request = Mock(return_value=response) - monkeypatch.setattr(requests.Session, "request", request) - return request - - -def test_extract_forwards_params_without_mutating_input(extract_request): - client = serpapi.Client(api_key="client-key", timeout=60) - params = { - "url": "https://example.com/?q=coffee&lang=en", - "engine": "google", - "mode": "map", - "include_markdown": False, - "timeout": 30000, - } - original = params.copy() - - result = client.extract(params, engine="bing", mode="crawl", max_pages=2) - - assert params == original - assert result["markdown"] == "Example content." - extract_request.assert_called_once() - _, sent = extract_request.call_args - assert sent["method"] == "GET" - assert sent["url"] == "https://serpapi.com/search" - assert sent["params"] == dict( - original, engine="extract", mode="crawl", max_pages=2, api_key="client-key" - ) - assert sent["timeout"] == 60 - - -def test_extract_separates_api_and_http_timeouts(extract_request): - client = serpapi.Client(api_key="client-key", timeout=60) - - client.extract( - {"url": "https://example.com/", "timeout": 30000}, - timeout=90, - ) - - _, sent = extract_request.call_args - assert sent["params"]["timeout"] == 30000 - assert sent["timeout"] == 90 - - -def test_module_extract_uses_request_key(extract_request): - result = serpapi.extract(url="https://example.com/", api_key="request-key") - - assert result["markdown"] == "Example content." - _, sent = extract_request.call_args - assert sent["params"] == { - "engine": "extract", - "url": "https://example.com/", - "api_key": "request-key", - } - - -@pytest.fixture -def extract_live_client(): - api_key = os.environ.get("API_KEY") or os.environ.get("SERPAPI_KEY") - if not api_key: - pytest.fail("Extract API live tests require API_KEY or SERPAPI_KEY") - client = serpapi.Client(api_key=api_key, timeout=180) - yield client - client.session.close() - - -def assert_success(result): - assert isinstance(result, serpapi.SerpResults) - assert "error" not in result - assert result["search_metadata"]["status"] == "Success" - - -def test_live_extract_scrape(extract_live_client): - result = extract_live_client.extract( - {"url": "https://serpapi.com/", "timeout": 30000}, - timeout=90, - ) - - assert_success(result) - assert "SerpApi" in result["page_information"]["title"] - assert result["page_information"]["final_url"] == "https://serpapi.com/" - assert isinstance(result["markdown"], str) - assert result["markdown"].strip() - - -def test_live_extract_map(extract_live_client): - result = extract_live_client.extract( - url="https://serpapi.com/", - mode="map", - include_sitemap=False, - include_subdomains=False, - ) - - assert_success(result) - urls = [entry["url"] for entry in result["urls"]] - assert urls - assert len(urls) == len(set(urls)) - assert all(urlparse(url).hostname == "serpapi.com" for url in urls) - assert all(urlparse(url).scheme in {"http", "https"} for url in urls) - - -def test_live_extract_crawl(extract_live_client): - result = extract_live_client.extract( - url="https://serpapi.com/", - mode="crawl", - max_pages=2, - max_depth=1, - ) - - assert_success(result) - pages = result["pages"] - assert len(pages) == 2 - assert pages[0]["url"] == "https://serpapi.com/" - assert pages[0]["depth"] == 0 - assert len({page["url"] for page in pages}) == 2 - for page in pages: - assert "error" not in page - assert page["markdown"] - assert page["depth"] <= 1 - - -@pytest.mark.parametrize("output", ["html", "md"]) -def test_live_extract_text(extract_live_client, output): - result = extract_live_client.extract(url="https://serpapi.com/", output=output) - - assert isinstance(result, str) - assert "SerpApi" in result - if output == "html": - assert "" in result.lower() - else: - assert "" in coffee_search_html.lower() - assert "coffee" in coffee_search_html.lower() - assert not hasattr(coffee_search_html, "next_page_url") - - -def test_next_page_url_uses_serpapi_pagination_next(coffee_search): - next_page_url = coffee_search.next_page_url - - assert next_page_url == coffee_search["serpapi_pagination"]["next"] - assert _query_param(next_page_url, "start") == "10" - assert _query_param(next_page_url, "api_key") is None - - -def test_yield_pages_returns_unique_search_pages(coffee_search): - max_pages = 3 - pages = list(coffee_search.yield_pages(max_pages=max_pages)) - - assert len(pages) == max_pages - assert len({page["search_metadata"]["id"] for page in pages}) == max_pages - assert "start" not in pages[0]["search_parameters"] - assert int(pages[1]["search_parameters"]["start"]) == 10 - assert int(pages[2]["search_parameters"]["start"]) == 20 - - -def test_next_page_advances_start_and_returns_new_results(coffee_search): - next_page = coffee_search.next_page() - - assert isinstance(next_page, serpapi.SerpResults) - assert coffee_search["search_metadata"]["id"] != next_page["search_metadata"]["id"] - assert int(next_page["search_parameters"]["start"]) == 10 - assert next_page["organic_results"] - - page_number = next_page["search_information"].get("page_number") - if page_number is not None: - assert int(page_number) == 2 - - -def test_search_archive_round_trips_search_id(client, coffee_search): - search_id = coffee_search["search_metadata"]["id"] - - archived_search = client.search_archive(search_id=search_id) - - assert isinstance(archived_search, serpapi.SerpResults) - assert archived_search["search_metadata"]["id"] == search_id - - -def test_search_function_signature(coffee_params, client): - s = client.search(coffee_params) - assert s["search_metadata"]["id"] - - s = client.search(**coffee_params) - assert s["search_metadata"]["id"] - - s = client.search(q='coffee') - assert s["search_metadata"]["id"] diff --git a/tests/test_output_formats.py b/tests/test_output_formats.py deleted file mode 100644 index ebafa9d..0000000 --- a/tests/test_output_formats.py +++ /dev/null @@ -1,52 +0,0 @@ -from unittest.mock import Mock - -import pytest -import requests - -import serpapi - - -def response(content_type, content): - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = f"{content_type}; charset=utf-8" - response._content = content.encode("utf-8") - return response - - -def test_search_json_returns_serp_results(): - client = serpapi.Client(api_key="test-api-key") - client.session.request = Mock( - return_value=response( - "application/json", - '{"search_metadata": {"id": "search-123"}}', - ) - ) - - result = client.search(engine="google", q="Coffee", output="json") - - assert isinstance(result, serpapi.SerpResults) - assert result["search_metadata"]["id"] == "search-123" - assert result.client is client - _, request_kwargs = client.session.request.call_args - assert request_kwargs["params"]["output"] == "json" - - -@pytest.mark.parametrize( - ("output", "content_type"), - [ - ("html", "text/html"), - ("md", "text/markdown"), - ], -) -def test_search_text_outputs_return_raw_text(output, content_type): - content = '{"looks": "like JSON"}' - client = serpapi.Client(api_key="test-api-key") - client.session.request = Mock(return_value=response(content_type, content)) - - result = client.search(engine="google", q="Coffee", output=output) - - assert result == content - assert isinstance(result, str) - _, request_kwargs = client.session.request.call_args - assert request_kwargs["params"]["output"] == output diff --git a/tests/test_pagination.py b/tests/test_pagination.py deleted file mode 100644 index 6fce28b..0000000 --- a/tests/test_pagination.py +++ /dev/null @@ -1,39 +0,0 @@ -import json -from unittest.mock import Mock - -import pytest -import requests - -import serpapi - - -@pytest.mark.parametrize( - ("max_pages", "available_pages", "expected_pages"), - [(1, 3, 1), (2, 3, 2), (3, 3, 3), (5, 3, 3), (5, 1, 1), (0, 3, 0)], -) -def test_yield_pages_does_not_request_unused_pages( - max_pages, available_pages, expected_pages -): - responses = [] - for page_number in range(1, available_pages + 1): - data = {"search_information": {"page_number": page_number}} - if page_number < available_pages: - data["serpapi_pagination"] = { - "next": f"https://serpapi.com/search?engine=google&q=Coffee&start={page_number * 10}" - } - response = requests.Response() - response.status_code = 200 - response.headers["Content-Type"] = "application/json" - response._content = json.dumps(data).encode("utf-8") - responses.append(response) - - client = serpapi.Client(api_key="test-api-key") - client.session.request = Mock(side_effect=responses) - results = client.search(engine="google", q="Coffee") - - pages = list(results.yield_pages(max_pages=max_pages)) - - assert [page["search_information"]["page_number"] for page in pages] == list( - range(1, expected_pages + 1) - ) - assert client.session.request.call_count == max(1, expected_pages) diff --git a/tests/test_timeout.py b/tests/test_timeout.py deleted file mode 100644 index 7ce5fbd..0000000 --- a/tests/test_timeout.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest -import requests -from serpapi import Client - -def test_client_timeout_setting(): - """Test that timeout can be set on the client and is passed to the request.""" - client = Client(api_key="test_key", timeout=10) - assert client.timeout == 10 - -def test_request_timeout_override(monkeypatch): - """Test that timeout can be overridden in the search method.""" - client = Client(api_key="test_key", timeout=10) - - def mock_request(method, url, params, headers, timeout, **kwargs): - assert timeout == 5 - # Return a mock response object - mock_response = requests.Response() - mock_response.status_code = 200 - mock_response._content = b'{"search_metadata": {"id": "123"}}' - return mock_response - - monkeypatch.setattr(client.session, "request", mock_request) - - client.search(q="coffee", timeout=5) - -def test_request_default_timeout(monkeypatch): - """Test that the client's default timeout is used if none is provided in search.""" - client = Client(api_key="test_key", timeout=10) - - def mock_request(method, url, params, headers, timeout, **kwargs): - assert timeout == 10 - mock_response = requests.Response() - mock_response.status_code = 200 - mock_response._content = b'{"search_metadata": {"id": "123"}}' - return mock_response - - monkeypatch.setattr(client.session, "request", mock_request) - - client.search(q="coffee")