fix: 修复下载数据重复 close #2718 (#2719)

* fix: 修复下载数据重复 close #2718 * fix: 修复透视表导出报错问题 * test: 提交单测 * refactor: 修改变量名 * refactor: 抽取公用变量、方法到 BaseDataCellCopy 基类 --------- Co-authored-by: Harvey Wang <[email protected]>
antvis · May 14, 2024 · f0d5192 · f0d5192
1 parent 185b59c
commit f0d5192
Show file tree

Hide file tree

Showing 6 changed files with 140 additions and 22 deletions.
diff --git a/packages/s2-core/__tests__/unit/utils/export/export-pivot-spec.ts b/packages/s2-core/__tests__/unit/utils/export/export-pivot-spec.ts
@@ -15,6 +15,7 @@ import {
   TAB_SEPARATOR,
 } from '../../../../src/common/constant';
 import { asyncGetAllPlainData, PivotSheet } from '../../../../src';
+import { generateRawData } from '../../../util';
 import { CopyMIMEType } from '@/common/interface/export';
 
 describe('PivotSheet Export Test', () => {
@@ -464,4 +465,40 @@ describe('PivotSheet Export Test', () => {
       expect(data.split(LINE_SEPARATOR)).toMatchSnapshot();
     },
   );
+
+  // https://github.com/antvis/S2/issues/2718
+  it('should export the correct amount of data and have no duplicate data', async () => {
+    const typeCount = 100;
+    const subTypeCount = 100;
+
+    const bigData = generateRawData(
+      { province: 10, city: 1 },
+      { type: typeCount, subType: subTypeCount },
+    );
+
+    const sheet = new PivotSheet(
+      getContainer(),
+      assembleDataCfg({
+        data: bigData,
+        fields: {
+          rows: ['type', 'subType'],
+          columns: ['province', 'city'],
+          values: ['number'],
+        },
+      }),
+      assembleOptions({}),
+    );
+
+    await sheet.render();
+    const data = await asyncGetAllPlainData({
+      sheetInstance: sheet,
+      split: CSV_SEPARATOR,
+      formatOptions: true,
+    });
+
+    // row header count + data count
+    const count = typeCount * subTypeCount + 3;
+
+    expect(data.split(LINE_SEPARATOR)).toHaveLength(count);
+  });
 });
diff --git a/packages/s2-core/__tests__/unit/utils/export/export-table-spec.ts b/packages/s2-core/__tests__/unit/utils/export/export-table-spec.ts
@@ -1,7 +1,11 @@
 /* eslint-disable jest/expect-expect */
 import { slice } from 'lodash';
 import { data as originData } from 'tests/data/mock-dataset.json';
-import { assembleDataCfg, assembleOptions } from '../../../util';
+import {
+  assembleDataCfg,
+  assembleOptions,
+  generateRawData,
+} from '../../../util';
 import {
   createTableSheet,
   expectMatchSnapshot,
@@ -433,4 +437,33 @@ describe('TableSheet Export Test', () => {
       formatData: false,
     });
   });
+
+  // https://github.com/antvis/S2/issues/2718
+  it('should export the correct amount of data and have no duplicate data', async () => {
+    const bigData = generateRawData(
+      { province: 10, city: 10 },
+      { type: 10, sub_type: 10 },
+    );
+
+    const tableSheet = new TableSheet(
+      getContainer(),
+      assembleDataCfg({
+        data: bigData,
+        fields: {
+          columns: ['province', 'city', 'type', 'sub_type', 'number'],
+        },
+      }),
+      assembleOptions(),
+    );
+
+    await tableSheet.render();
+    const data = await asyncGetAllPlainData({
+      sheetInstance: tableSheet,
+      split: CSV_SEPARATOR,
+      formatOptions: true,
+    });
+
+    // The first line is the header, so the number of lines should be the same as the number of data plus one
+    expect(data.split(LINE_SEPARATOR)).toHaveLength(bigData.length + 1);
+  });
 });
diff --git a/packages/s2-core/__tests__/util/index.ts b/packages/s2-core/__tests__/util/index.ts
@@ -34,6 +34,34 @@ export const assembleDataCfg = (...dataCfg: Partial<S2DataConfig>[]) => {
   return customMerge<S2DataConfig>(DEFAULT_DATA_CONFIG, s2DataCfg, ...dataCfg);
 };
 
+export const generateRawData = (
+  row: Record<string, number>,
+  col: Record<string, number>,
+) => {
+  const res: Record<string, any>[] = [];
+
+  const rowKeys = Object.keys(row);
+  const colKeys = Object.keys(col);
+
+  for (let i = 0; i < row[rowKeys[0]]; i++) {
+    for (let j = 0; j < row[rowKeys[1]]; j++) {
+      for (let m = 0; m < col[colKeys[0]]; m++) {
+        for (let n = 0; n < col[colKeys[1]]; n++) {
+          res.push({
+            province: `p${i}`,
+            city: `c${j}`,
+            type: `type${m}`,
+            subType: `subType${n}`,
+            number: i * n,
+          });
+        }
+      }
+    }
+  }
+
+  return res;
+};
+
 export const TOTALS_OPTIONS: S2Options['totals'] = {
   row: {
     showGrandTotals: true,

diff --git a/packages/s2-core/src/utils/export/copy/base-data-cell-copy.ts b/packages/s2-core/src/utils/export/copy/base-data-cell-copy.ts
@@ -1,4 +1,8 @@
-import { TAB_SEPARATOR, type DataItem } from '../../../common';
+import {
+  AsyncRenderThreshold,
+  TAB_SEPARATOR,
+  type DataItem,
+} from '../../../common';
 import type {
   CopyableHTML,
   CopyablePlain,
@@ -16,6 +20,13 @@ export abstract class BaseDataCellCopy {
 
   protected config: CopyAndExportUnifyConfig;
 
+  protected idleCallbackCount: number;
+
+  protected initIdleCallbackCount(rowLength: number) {
+    this.idleCallbackCount =
+      rowLength >= AsyncRenderThreshold ? AsyncRenderThreshold : rowLength;
+  }
+
   constructor(params: SheetCopyConstructorParams) {
     const { spreadsheet, isExport = false, config } = params;
 

diff --git a/packages/s2-core/src/utils/export/copy/pivot-data-cell-copy.ts b/packages/s2-core/src/utils/export/copy/pivot-data-cell-copy.ts
@@ -8,7 +8,6 @@ import {
   zip,
 } from 'lodash';
 import {
-  AsyncRenderThreshold,
   CornerNodeType,
   EXTRA_FIELD,
   VALUE_FIELD,
@@ -153,18 +152,20 @@ export class PivotDataCellCopy extends BaseDataCellCopy {
         // 因为每次 requestIdleCallback 执行的时间不一样，所以需要记录下当前执行到的 this.leafRowNodes 和 this.leafColNodes
         const dataMatrixIdleCallback = (deadline: IdleDeadline) => {
           const rowLength: number = this.leafRowNodes.length;
+
           // requestIdleCallback 浏览器空闲时会多次执行, 只有一行数据时执行一次即可, 避免生成重复数据
-          let count =
-            rowLength >= AsyncRenderThreshold
-              ? AsyncRenderThreshold
-              : rowLength;
+          this.initIdleCallbackCount(rowLength);
 
           while (
             (deadline.timeRemaining() > 0 || deadline.didTimeout) &&
             rowIndex <= rowLength - 1 &&
-            count > 0
+            this.idleCallbackCount > 0
           ) {
-            for (let j = rowIndex; j < rowLength && count > 0; j++) {
+            for (
+              let j = rowIndex;
+              j < rowLength && this.idleCallbackCount > 0;
+              j++
+            ) {
               const row: DataItem[] = [];
               const rowNode = this.leafRowNodes[j];
 
@@ -183,15 +184,18 @@ export class PivotDataCellCopy extends BaseDataCellCopy {
 
                 row.push(dataItem);
               }
-              rowIndex = j;
+              rowIndex++;
               matrix.push(row);
-              count--;
+              this.idleCallbackCount--;
             }
           }
 
-          if (rowIndex === rowLength - 1) {
+          if (rowIndex === rowLength) {
             resolve(matrix);
           } else {
+            // 重置 idleCallbackCount，避免下次 requestIdleCallback 时 idleCallbackCount 为 0
+            this.initIdleCallbackCount(rowLength);
+
             requestIdleCallback(dataMatrixIdleCallback);
           }
         };

diff --git a/packages/s2-core/src/utils/export/copy/table-copy.ts b/packages/s2-core/src/utils/export/copy/table-copy.ts
@@ -1,6 +1,5 @@
 import { map, zip } from 'lodash';
 import {
-  AsyncRenderThreshold,
   SERIES_NUMBER_FIELD,
   getDefaultSeriesNumberText,
   type CellMeta,
@@ -94,20 +93,22 @@ class TableDataCellCopy extends BaseDataCellCopy {
       try {
         const dataMatrixIdleCallback = (deadline: IdleDeadline) => {
           const rowLength = this.displayData.length;
+
           // requestIdleCallback 浏览器空闲时会多次执行, 只有一行数据时执行一次即可, 避免生成重复数据
-          let count =
-            rowLength >= AsyncRenderThreshold
-              ? AsyncRenderThreshold
-              : rowLength;
+          this.initIdleCallbackCount(rowLength);
 
           while (
             (deadline.timeRemaining() > 0 ||
               deadline.didTimeout ||
               process.env['NODE_ENV'] === 'test') &&
             rowIndex <= rowLength - 1 &&
-            count > 0
+            this.idleCallbackCount > 0
           ) {
-            for (let j = rowIndex; j < rowLength && count > 0; j++) {
+            for (
+              let j = rowIndex;
+              j < rowLength && this.idleCallbackCount > 0;
+              j++
+            ) {
               const rowData = this.displayData[j];
               const row: string[] = [];
 
@@ -131,15 +132,19 @@ class TableDataCellCopy extends BaseDataCellCopy {
 
                 row.push(dataItem as string);
               }
-              rowIndex = j;
+              // 生成一行数据后，rowIndex + 1，下次 requestIdleCallback 时从下一行开始
+              rowIndex++;
               result.push(row);
-              count--;
+              this.idleCallbackCount--;
             }
           }
 
-          if (rowIndex === rowLength - 1) {
+          if (rowIndex === rowLength) {
             resolve(result);
           } else {
+            // 重置 idleCallbackCount，避免下次 requestIdleCallback 时 idleCallbackCount 为 0
+            this.initIdleCallbackCount(rowLength);
+
             requestIdleCallback(dataMatrixIdleCallback);
           }
         };