Skip to content
StudioCMS Docs
Main Site

lib/robots

Functions

Section titled “Functions”

default()

Section titled “default()”
function default(options: {
  host: string | boolean;
  policy: {
     allow: string | string[];
     cleanParam: string | string[];
     crawlDelay: number;
     disallow: string | string[];
     userAgent:   | "*"
        | "360Spider"
        | "360Spider-Image"
        | "360Spider-Video"
        | "HaoSouSpider"
        | "AdsBot-Google"
        | "AdsBot-Google-Mobile"
        | "AdsBot-Google-Mobile-Apps"
        | "Googlebot"
        | "Googlebot-Image"
        | "Googlebot-Mobile"
        | "Googlebot-News"
        | "Googlebot-Video"
        | "Mediapartners-Google"
        | "adidxbot"
        | "bingbot"
        | "BingPreview"
        | "MicrosoftPreview"
        | "msnbot"
        | "msnbot-media"
        | "Applebot"
        | "AppleNewsBot"
        | "Baiduspider"
        | "Baiduspider-image"
        | "Baiduspider-mobile"
        | "Baiduspider-news"
        | "Baiduspider-video"
        | "coccoc"
        | "coccocbot-image"
        | "coccocbot-web"
        | "DuckDuckBot"
        | "DuckDuckGo-Favicons-Bot"
        | "facebookcatalog"
        | "facebookexternalhit"
        | "Facebot"
        | "gooblog"
        | "ichiro"
        | "Sogou blog"
        | "Sogou inst spider"
        | "Sogou News Spider"
        | "Sogou Orion spider"
        | "Sogou spider2"
        | "Sogou web spider"
        | "Yandex"
        | "YandexMobileBot"
        | "Algolia Crawler"
        | "BublupBot"
        | "CCBot"
        | "Cliqzbot"
        | "Daumoa"
        | "DeuSu"
        | "EuripBot"
        | "Exploratodo"
        | "Feedly"
        | "Findxbot"
        | "istellabot"
        | "JikeSpider"
        | "Lycos"
        | "Mail.Ru"
        | "MojeekBot"
        | "OrangeBot"
        | "Pinterest"
        | "Plukkie"
        | "Qwantify"
        | "Rambler"
        | "SemanticScholarBot"
        | "SeznamBot"
        | "Sosospider"
        | "Slurp"
        | "Twitterbot"
        | "WhatsApp"
        | "yacybot"
        | "YepBot"
        | "Yeti"
        | "YioopBot"
        | "yoozBot"
        | "YoudaoBot"
        | (
        | "*"
        | "360Spider"
        | "360Spider-Image"
        | "360Spider-Video"
        | "HaoSouSpider"
        | "AdsBot-Google"
        | "AdsBot-Google-Mobile"
        | "AdsBot-Google-Mobile-Apps"
        | "Googlebot"
        | "Googlebot-Image"
        | "Googlebot-Mobile"
        | "Googlebot-News"
        | "Googlebot-Video"
        | "Mediapartners-Google"
        | "adidxbot"
        | "bingbot"
        | "BingPreview"
        | "MicrosoftPreview"
        | "msnbot"
        | "msnbot-media"
        | "Applebot"
        | "AppleNewsBot"
        | "Baiduspider"
        | "Baiduspider-image"
        | "Baiduspider-mobile"
        | "Baiduspider-news"
        | "Baiduspider-video"
        | "coccoc"
        | "coccocbot-image"
        | "coccocbot-web"
        | "DuckDuckBot"
        | "DuckDuckGo-Favicons-Bot"
        | "facebookcatalog"
        | "facebookexternalhit"
        | "Facebot"
        | "gooblog"
        | "ichiro"
        | "Sogou blog"
        | "Sogou inst spider"
        | "Sogou News Spider"
        | "Sogou Orion spider"
        | "Sogou spider2"
        | "Sogou web spider"
        | "Yandex"
        | "YandexMobileBot"
        | "Algolia Crawler"
        | "BublupBot"
        | "CCBot"
        | "Cliqzbot"
        | "Daumoa"
        | "DeuSu"
        | "EuripBot"
        | "Exploratodo"
        | "Feedly"
        | "Findxbot"
        | "istellabot"
        | "JikeSpider"
        | "Lycos"
        | "Mail.Ru"
        | "MojeekBot"
        | "OrangeBot"
        | "Pinterest"
        | "Plukkie"
        | "Qwantify"
        | "Rambler"
        | "SemanticScholarBot"
        | "SeznamBot"
        | "Sosospider"
        | "Slurp"
        | "Twitterbot"
        | "WhatsApp"
        | "yacybot"
        | "YepBot"
        | "Yeti"
        | "YioopBot"
        | "yoozBot"
        | "YoudaoBot")[];
    }[];
  sitemap: string | boolean | string[];
 }): AstroIntegration

Defined in: studiocms/packages/studiocms/src/lib/robots/index.ts:16^

Robots.txt Integration

A simple integration to generate a robots.txt file for your Astro project.

Parameters

Section titled “Parameters”
options
Section titled “options”
host?
Section titled “host?”

string | boolean = ...

Default

false

Description

[ Optional ] Some crawlers(Yandex) support and only accept domain names.

Example

integrations:[
 robots({
   host: siteUrl.replace(/^https?://|:\d+/g, "")
 })
]
policy?
Section titled “policy?”

{ allow: string | string[]; cleanParam: string | string[]; crawlDelay: number; disallow: string | string[]; userAgent: | "*" | "360Spider" | "360Spider-Image" | "360Spider-Video" | "HaoSouSpider" | "AdsBot-Google" | "AdsBot-Google-Mobile" | "AdsBot-Google-Mobile-Apps" | "Googlebot" | "Googlebot-Image" | "Googlebot-Mobile" | "Googlebot-News" | "Googlebot-Video" | "Mediapartners-Google" | "adidxbot" | "bingbot" | "BingPreview" | "MicrosoftPreview" | "msnbot" | "msnbot-media" | "Applebot" | "AppleNewsBot" | "Baiduspider" | "Baiduspider-image" | "Baiduspider-mobile" | "Baiduspider-news" | "Baiduspider-video" | "coccoc" | "coccocbot-image" | "coccocbot-web" | "DuckDuckBot" | "DuckDuckGo-Favicons-Bot" | "facebookcatalog" | "facebookexternalhit" | "Facebot" | "gooblog" | "ichiro" | "Sogou blog" | "Sogou inst spider" | "Sogou News Spider" | "Sogou Orion spider" | "Sogou spider2" | "Sogou web spider" | "Yandex" | "YandexMobileBot" | "Algolia Crawler" | "BublupBot" | "CCBot" | "Cliqzbot" | "Daumoa" | "DeuSu" | "EuripBot" | "Exploratodo" | "Feedly" | "Findxbot" | "istellabot" | "JikeSpider" | "Lycos" | "Mail.Ru" | "MojeekBot" | "OrangeBot" | "Pinterest" | "Plukkie" | "Qwantify" | "Rambler" | "SemanticScholarBot" | "SeznamBot" | "Sosospider" | "Slurp" | "Twitterbot" | "WhatsApp" | "yacybot" | "YepBot" | "Yeti" | "YioopBot" | "yoozBot" | "YoudaoBot" | ( | "*" | "360Spider" | "360Spider-Image" | "360Spider-Video" | "HaoSouSpider" | "AdsBot-Google" | "AdsBot-Google-Mobile" | "AdsBot-Google-Mobile-Apps" | "Googlebot" | "Googlebot-Image" | "Googlebot-Mobile" | "Googlebot-News" | "Googlebot-Video" | "Mediapartners-Google" | "adidxbot" | "bingbot" | "BingPreview" | "MicrosoftPreview" | "msnbot" | "msnbot-media" | "Applebot" | "AppleNewsBot" | "Baiduspider" | "Baiduspider-image" | "Baiduspider-mobile" | "Baiduspider-news" | "Baiduspider-video" | "coccoc" | "coccocbot-image" | "coccocbot-web" | "DuckDuckBot" | "DuckDuckGo-Favicons-Bot" | "facebookcatalog" | "facebookexternalhit" | "Facebot" | "gooblog" | "ichiro" | "Sogou blog" | "Sogou inst spider" | "Sogou News Spider" | "Sogou Orion spider" | "Sogou spider2" | "Sogou web spider" | "Yandex" | "YandexMobileBot" | "Algolia Crawler" | "BublupBot" | "CCBot" | "Cliqzbot" | "Daumoa" | "DeuSu" | "EuripBot" | "Exploratodo" | "Feedly" | "Findxbot" | "istellabot" | "JikeSpider" | "Lycos" | "Mail.Ru" | "MojeekBot" | "OrangeBot" | "Pinterest" | "Plukkie" | "Qwantify" | "Rambler" | "SemanticScholarBot" | "SeznamBot" | "Sosospider" | "Slurp" | "Twitterbot" | "WhatsApp" | "yacybot" | "YepBot" | "Yeti" | "YioopBot" | "yoozBot" | "YoudaoBot")[]; }[] = ...

Description

[ Optional ] List of policy rules.

Default

policy:[
 {
   userAgent: "*",
   allow: "/"
 }
]

For more help, refer to SYNTAX^ by Yandex.

sitemap?
Section titled “sitemap?”

string | boolean | string[] = ...

Description

[ Optional, zero or more per file ] The location of a sitemap for this website.

Example

sitemap: [
 "https://example.com/sitemap.xml",
 "https://www.example.com/sitemap.xml"
]

The value of the SITEMAP^ field is case-sensitive.

Returns

Section titled “Returns”

AstroIntegration

AstroIntegration