RFC9309 · Robots Exclusion Protocol

{
  "id": "RFC9309",
  "number": 9309,
  "title_en": "Robots Exclusion Protocol",
  "title_ru": "",
  "summary_ru": "",
  "purpose_ru": "",
  "use_ru": "",
  "full_translation_path": "",
  "full_translation_checked_at": "",
  "full_translation_source_sha256": "",
  "architect_note_ru": "",
  "topics_ru": [],
  "translation_state": "not_prepared",
  "verification_level": "not_prepared",
  "verification_level_ru": "Русская выжимка не подготовлена",
  "verification_note_ru": "",
  "abstract_en": "This document specifies and extends the \"Robots Exclusion Protocol\" method originally defined by Martijn Koster in 1994 for service owners to control how content served by their services may be accessed, if at all, by automatic clients known as crawlers. Specifically, it adds definition language for the protocol, instructions for handling errors, and instructions for caching.",
  "authors": [
    "M. Koster",
    "G. Illyes",
    "H. Zeller",
    "L. Sassman"
  ],
  "organizations": [],
  "publication_date": "2022-09-01",
  "current_status": "PROPOSED STANDARD",
  "current_status_ru": "Предлагаемый стандарт",
  "publication_status": "PROPOSED STANDARD",
  "publication_status_ru": "Предлагаемый стандарт",
  "lifecycle": "active",
  "lifecycle_ru": "Действующий",
  "stream": "IETF",
  "stream_ru": "IETF",
  "area": "",
  "area_ru": "",
  "working_group": "NON WORKING GROUP",
  "source_draft": "draft-koster-rep-12",
  "page_count": 12,
  "doi": "10.17487/RFC9309",
  "keywords": [
    "robot",
    "crawler",
    "robots.txt"
  ],
  "formats": [
    "HTML",
    "TXT",
    "PDF",
    "XML"
  ],
  "relations": {
    "obsoletes": [],
    "obsoleted_by": [],
    "updates": [],
    "updated_by": [],
    "is_also": [],
    "see_also": []
  },
  "links": {
    "info": "https://www.rfc-editor.org/info/rfc9309/",
    "html": "https://www.rfc-editor.org/rfc/rfc9309.html",
    "txt": "https://www.rfc-editor.org/rfc/rfc9309.txt",
    "datatracker": "https://datatracker.ietf.org/doc/rfc9309/"
  }
}