网页爬虫检测办法

核心逻辑如下：

在服务器返回的页面中插入一段只有浏览器才能执行的代码，如果是正常浏览器请求，那么这段代码就会被执行，如果是爬虫获取到的该页面，那么这个爬虫大概率不会执行这段代码

有了这个思路，然后再结合后端数据库的存储，就可以实现一个爬虫检测的逻辑了

数据库设计

首先得有一个记录访问的表，暂且叫 visit 表，表里面应该有两个字段

• executed：表示该页面是否被执行过爬虫检查代码，1 表示被执行过，0（默认）表示没有被执行过
• normal：表示请求是否正常，1 表示正常，0（默认）表示爬虫

前端判断逻辑

<!-- 隐藏的陷阱元素 -->
<div style="position: absolute; left: -9999px; top: -9999px;">
  <a href="/" id="t-l" style="color: transparent; font-size: 0;">t-l</a>
  <form id="t-f" action="/" method="POST">
    <input type="hidden" name="trap" value="1" />
  </form>
</div>

<script>
  (function () {
    class BrowserValidator {
      constructor() {
        // 存储检测结果
        this.results = {
          basicEnv: false,
          advancedAPIs: false,
          behaviorPattern: false,
          fingerprintConsistency: false,
          trapAvoidance: false,
        };

        // 设置陷阱
        this.setupTraps();
      }

      // 基础环境检测
      checkBasicEnvironment() {
        const requiredObjects = [
          "window",
          "document",
          "navigator",
          "screen",
          "history",
          "location",
        ];

        const exists = requiredObjects.every((obj) => {
          try {
            return eval(`typeof ${obj} !== 'undefined'`);
          } catch (e) {
            return false;
          }
        });

        this.results.basicEnv = exists;
        return exists;
      }

      // 高级API检测
      checkAdvancedAPIs() {
        const apis = [
          "WebGLRenderingContext",
          "AudioContext",
          "BatteryManager",
          "RTCPeerConnection",
          "PushManager",
        ];

        let detectedCount = 0;

        // 检测WebGL
        try {
          const canvas = document.createElement("canvas");
          const gl =
            canvas.getContext("webgl") ||
            canvas.getContext("experimental-webgl");
          if (gl) detectedCount++;
        } catch (e) {}

        // 检测AudioContext
        if (window.AudioContext || window.webkitAudioContext) detectedCount++;

        // 检测Battery API
        if ("getBattery" in navigator) detectedCount++;

        // 检测WebRTC
        if (window.RTCPeerConnection) detectedCount++;

        // 检测Push API
        if ("PushManager" in window) detectedCount++;

        this.results.advancedAPIs = detectedCount >= 3;
        return detectedCount >= 3;
      }

      // 行为模式检测
      checkBehaviorPattern() {
        return new Promise((resolve) => {
          let mouseMoveDetected = false;
          let clickDetected = false;

          const onMouseMove = () => {
            mouseMoveDetected = true;
            document.removeEventListener("mousemove", onMouseMove);
          };

          const onClick = () => {
            clickDetected = true;
            document.removeEventListener("click", onClick);
          };

          document.addEventListener("mousemove", onMouseMove);
          document.addEventListener("click", onClick);

          // 等待用户交互
          setTimeout(() => {
            this.results.behaviorPattern = mouseMoveDetected || clickDetected;
            resolve(this.results.behaviorPattern);
          }, 2000);
        });
      }

      // 指纹一致性检测
      checkFingerprintConsistency() {
        try {
          // Canvas指纹检测
          const canvas = document.createElement("canvas");
          canvas.width = 200;
          canvas.height = 40;
          const ctx = canvas.getContext("2d");

          // 绘制文本
          ctx.textBaseline = "top";
          ctx.font = "14px Arial";
          ctx.fillStyle = "#f60";
          ctx.fillRect(0, 0, canvas.width, canvas.height);
          ctx.fillStyle = "#069";
          ctx.fillText("Browser Validator", 5, 5);

          // 获取图像数据
          const dataUrl = canvas.toDataURL();

          // 简单验证数据是否正常（真实浏览器会产生唯一指纹）
          this.results.fingerprintConsistency = dataUrl.length > 1000;
          return this.results.fingerprintConsistency;
        } catch (e) {
          this.results.fingerprintConsistency = false;
          return false;
        }
      }

      // 设置陷阱
      setupTraps() {
        // 标记陷阱是否被触发
        this.trapTriggered = false;

        // 监听隐藏链接的点击
        document.getElementById("t-l").addEventListener("click", (e) => {
          e.preventDefault();
          this.trapTriggered = true;
        });

        // 监听表单提交
        document.getElementById("t-f").addEventListener("submit", (e) => {
          e.preventDefault();
          this.trapTriggered = true;
        });
      }

      // 陷阱检测
      checkTrapAvoidance() {
        this.results.trapAvoidance = !this.trapTriggered;
        return this.results.trapAvoidance;
      }

      // 综合检测
      async validate() {
        // 执行同步检测
        this.checkBasicEnvironment();
        this.checkAdvancedAPIs();
        this.checkFingerprintConsistency();
        this.checkTrapAvoidance();

        // 执行异步检测
        await this.checkBehaviorPattern();

        // 计算综合结果
        const passedTests = Object.values(this.results).filter(Boolean).length;
        const totalTests = Object.keys(this.results).length;

        return {
          isBrowser: passedTests >= 4, // 通过4项及以上测试
          score: `${passedTests}/${totalTests}`,
          details: this.results,
        };
      }
    }

    // 页面加载完成后执行检测
    document.addEventListener("DOMContentLoaded", async () => {
      const validator = new BrowserValidator();
      const result = await validator.validate();

      // 这里将爬虫判断结果通过verifySpider函数返回给后端，关键是将isBrowser的赋给visit表的normal字段
      if (verifySpider instanceof Function) verifySpider(result);
      console.log(result);
    });
  })();
</script>

如果为了避免被爬虫发现这段脚本，可以压缩丑化掉这段代码，具体压缩工具可以自行选择，这里推荐使用 HTML/JS 兼容性转换与压缩工具