lib/robots/types

Interfaces

PolicyOptions

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:180

Properties

allow?

optional allow: string | string[];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:210

Description

[ At least one or more allow or disallow entries per rule ] Allows indexing site sections or individual pages.

Example

policy:[{allow:["/"]}]

Path-based URL matching, refer to SYNTAX^ via Google.

cleanParam?

optional cleanParam: string | string[];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:261

Description

[ Optional ] Indicates to the robot that the page URL contains parameters (like UTM tags) that should be ignored when indexing it.

Example

# for URLs like:
www.example2.com/index.php?page=1&sid=2564126ebdec301c607e5df
www.example2.com/index.php?page=1&sid=974017dcd170d6c4a5d76ae

policy:[
 {
   cleanParam: [
     "sid /index.php",
   ]
 }
]

For additional examples, please consult Yandex’s SYNTAX^ guide.

crawlDelay?

optional crawlDelay: number;

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:238

Description

[ Optional ] Specifies the minimum interval (in seconds) for the search robot to wait after loading one page, before starting to load another.

Example

policy:[{crawlDelay:5}]

About the Crawl-delay^ directive.

disallow?

optional disallow: string | string[];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:227

Description

[ At least one or more disallow or allow entries per rule ] Prohibits indexing site sections or individual pages.

Example

policy:[
 {
   disallow:[
     "/admin",
     "/uploads/1989-08-21/*.jpg$"
   ]
 }
]

Path-based URL matching, refer to SYNTAX^ via Google.

userAgent?

optional userAgent:
  | UserAgentType
  | UserAgentType[];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:200

Description

[ Required ] Indicates the robot to which the rules listed in robots.txt apply.

Example

policy:[
 {
   userAgent: [
     'Googlebot',
     'Applebot',
     'Baiduspider',
     'bingbot'
   ],
   // crawling rule(s) for above bots
 }
]

Verified bots, refer to DITIG^ or Cloudflare Radar^.

RobotsConfig

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:135

Properties

host?

optional host: string | boolean;

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:149

Default

false

Description

[ Optional ] Some crawlers(Yandex) support and only accept domain names.

Example

integrations:[
 robots({
   host: siteUrl.replace(/^https?://|:\d+/g, "")
 })
]

policy?

optional policy: PolicyOptions[];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:177

Description

[ Optional ] List of policy rules.

Default

policy:[
 {
   userAgent: "*",
   allow: "/"
 }
]

For more help, refer to SYNTAX^ by Yandex.

sitemap?

optional sitemap: string | boolean | string[];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:162

Description

[ Optional, zero or more per file ] The location of a sitemap for this website.

Example

sitemap: [
 "https://example.com/sitemap.xml",
 "https://www.example.com/sitemap.xml"
]

The value of the SITEMAP^ field is case-sensitive.

Type Aliases

SearchEngineOptimization

type SearchEngineOptimization = {
  Ahrefs: "AhrefsBot";
  Babbar: "Barkrowler";
  Botify: "Botify";
  Cxense: "Cxense";
  DataForSEO: "DataForSEO";
  EzoicInc: "EzoicBot";
  Moz: "Moz dotbot" | "Moz rogerbot";
  PrerenderLLC: "prerender";
  SEMrush: "SEMrush" | "SemrushBotSI";
  WebMeUp: "BLEXBot";
};

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:116

Type declaration

Ahrefs

Ahrefs: "AhrefsBot";

Babbar

Babbar: "Barkrowler";

Botify

Botify: "Botify";

Cxense

Cxense: "Cxense";

DataForSEO

DataForSEO: "DataForSEO";

EzoicInc

EzoicInc: "EzoicBot";

Moz

Moz: "Moz dotbot" | "Moz rogerbot";

PrerenderLLC

PrerenderLLC: "prerender";

SEMrush

SEMrush: "SEMrush" | "SemrushBotSI";

WebMeUp

WebMeUp: "BLEXBot";

SearchEngines

type SearchEngines = {
  360: "360Spider" | "360Spider-Image" | "360Spider-Video" | "HaoSouSpider";
  Apple: "Applebot" | "AppleNewsBot";
  Ask: "Teoma";
  Baidu:   | "Baiduspider"
     | "Baiduspider-image"
     | "Baiduspider-mobile"
     | "Baiduspider-news"
     | "Baiduspider-video";
  Bing:   | "bingbot"
     | "BingPreview"
     | "msnbot"
     | "msnbot-media"
     | "adidxbot"
     | "MSN";
  Botje: "Plukkie";
  Bublup: "BublupBot";
  Cliqz: "Cliqzbot";
  Coccoc: "coccoc" | "coccocbot-image" | "coccocbot-web";
  Daumoa: "Daumoa";
  Dazoo: "DeuSu";
  Duckduckgo: "DuckDuckBot" | "DuckDuckGo-Favicons-Bot";
  Eurip: "EuripBot";
  Exploratodo: "Exploratodo";
  Findx: "Findxbot";
  Goo: "gooblog" | "ichiro";
  Google:   | "Googlebot"
     | "Googlebot-Image"
     | "Googlebot-Mobile"
     | "Googlebot-News"
     | "Googlebot-Video"
     | "Mediapartners-Google"
     | "AdsBot-Google"
     | "AdsBot-Google-Mobile"
     | "AdsBot-Google-Mobile-Apps"
     | "Mediapartners-Google"
     | "Storebot-Google"
     | "Google-InspectionTool"
     | "FeedFetcher-Google";
  Istella: "istellabot";
  Jike: "JikeSpider";
  Lycos: "Lycos";
  Mail: "Mail.Ru";
  Mojeek: "MojeekBot";
  Naver: "Yeti";
  Orange: "OrangeBot";
  Qwant: "Qwantify";
  Rambler: "Rambler";
  Seznam: "SeznamBot";
  Sogou:   | "Sogou blog"
     | "Sogou inst spider"
     | "Sogou News Spider"
     | "Sogou Orion spider"
     | "Sogou spider2"
     | "Sogou web spider";
  Soso: "Sosospider";
  Sputnik: "SputnikBot";
  Wortbox: "wotbox";
  Yahoo: "Slurp";
  Yandex: "Yandex" | "YandexMobileBot";
  Yioop: "YioopBot";
  Yooz: "yoozBot";
  Youdao: "YoudaoBot";
};

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:4

Type declaration

360

360: "360Spider" | "360Spider-Image" | "360Spider-Video" | "HaoSouSpider";

Apple

Apple: "Applebot" | "AppleNewsBot";

Ask

Ask: "Teoma";

Baidu

Baidu:
  | "Baiduspider"
  | "Baiduspider-image"
  | "Baiduspider-mobile"
  | "Baiduspider-news"
  | "Baiduspider-video";

Bing

Bing:
  | "bingbot"
  | "BingPreview"
  | "msnbot"
  | "msnbot-media"
  | "adidxbot"
  | "MSN";

Botje

Botje: "Plukkie";

Bublup

Bublup: "BublupBot";

Cliqz

Cliqz: "Cliqzbot";

Coccoc

Coccoc: "coccoc" | "coccocbot-image" | "coccocbot-web";

Daumoa

Daumoa: "Daumoa";

Dazoo

Dazoo: "DeuSu";

Duckduckgo

Duckduckgo: "DuckDuckBot" | "DuckDuckGo-Favicons-Bot";

Eurip

Eurip: "EuripBot";

Exploratodo

Exploratodo: "Exploratodo";

Findx

Findx: "Findxbot";

Goo

Goo: "gooblog" | "ichiro";

Google

Google:
  | "Googlebot"
  | "Googlebot-Image"
  | "Googlebot-Mobile"
  | "Googlebot-News"
  | "Googlebot-Video"
  | "Mediapartners-Google"
  | "AdsBot-Google"
  | "AdsBot-Google-Mobile"
  | "AdsBot-Google-Mobile-Apps"
  | "Mediapartners-Google"
  | "Storebot-Google"
  | "Google-InspectionTool"
  | "FeedFetcher-Google";

Istella

Istella: "istellabot";

Jike

Jike: "JikeSpider";

Lycos

Lycos: "Lycos";

Mail

Mail: "Mail.Ru";

Mojeek

Mojeek: "MojeekBot";

Naver

Naver: "Yeti";

Orange

Orange: "OrangeBot";

Qwant

Qwant: "Qwantify";

Rambler

Rambler: "Rambler";

Seznam

Seznam: "SeznamBot";

Sogou

Sogou:
  | "Sogou blog"
  | "Sogou inst spider"
  | "Sogou News Spider"
  | "Sogou Orion spider"
  | "Sogou spider2"
  | "Sogou web spider";

Soso

Soso: "Sosospider";

Sputnik

Sputnik: "SputnikBot";

Wortbox

Wortbox: "wotbox";

Yahoo

Yahoo: "Slurp";

Yandex

Yandex: "Yandex" | "YandexMobileBot";

Yioop

Yioop: "YioopBot";

Yooz

Yooz: "yoozBot";

Youdao

Youdao: "YoudaoBot";

SocialNetwork

type SocialNetwork = {
  Facebook: "facebookcatalog" | "facebookexternalhit" | "Facebot";
  LinkedIn: "LinkedInBot";
  Pinterest: "Pinterest";
  Tittwer: "Twitterbot";
  WhatsApp: "WhatsApp";
};

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:103

Type declaration

Facebook

Facebook: "facebookcatalog" | "facebookexternalhit" | "Facebot";

LinkedIn: "LinkedInBot";

Pinterest: "Pinterest";

Tittwer

Tittwer: "Twitterbot";

WhatsApp: "WhatsApp";

UserAgentType

type UserAgentType =
  | "*"
  | SearchEngines[keyof SearchEngines]
  | SocialNetwork[keyof SocialNetwork]
  | SearchEngineOptimization[keyof SearchEngineOptimization];

Defined in: studiocms/packages/studiocms/src/lib/robots/types.ts:129