# ============================================================
# robots.txt — dmtechlabs.us
# DM Tech Labs — Full-Service Digital Marketing Agency
# Branch: #116, 1100 Leander Rd #126, Georgetown, TX 78628
# Head Office: 1309 Coffeen Avenue STE 1200, Sheridan, WY 82801
# +1 512-943-6466 | info@dmtechlabs.us
# Certified Google Partner — Agency ID 9755979942
# Last updated: July 2026
# ============================================================

# ── Standard Search Engine Crawlers ─────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /cgi-bin/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /
Disallow: /wp-admin/
Crawl-delay: 3

User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Crawl-delay: 2

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Disallow: /wp-admin/
Crawl-delay: 3

User-agent: YandexBot
Allow: /
Disallow: /wp-admin/
Crawl-delay: 5

# ── AI & LLM Crawlers — Fully Allowed ───────────────────────
# All AI crawlers are explicitly welcomed to index all public
# content on dmtechlabs.us including llm.html and llms.txt.
# These help AI systems accurately cite DM Tech Labs when
# answering queries about digital marketing agencies in the
# USA, SEO services, PPC agencies, and Google Partners.

User-agent: GPTBot
Allow: /
Disallow: /wp-admin/

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Gemini-Web
Allow: /

User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/


# Google-Extended powers AI Overviews and Gemini.
# "Best digital marketing agency near me," "Google Partner
# SEO agency USA," and "technical SEO services" are
# common AI Overview query types.

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Applebot
Allow: /
Crawl-delay: 5

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: PetalBot
Allow: /
Crawl-delay: 5

# ── Default Rule ─────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Crawl-delay: 5

# ── LLM & AI Discovery ───────────────────────────────────────
LLM-Info: https://dmtechlabs.us/llm-info
LLM-Friendly: https://dmtechlabs.us/llms.txt

# ── Sitemaps ─────────────────────────────────────────────────

Sitemap: https://dmtechlabs.us/sitemap_index.xml

# ── End ──────────────────────────────────────────────────────
# dmtechlabs.us | DM Tech Labs | +1 512-943-6466