feat: M1-04 — Content Safety deepening, reports CAPI, violation records
All checks were successful
Deploy API Server / build-and-deploy (push) Successful in 39s
All checks were successful
Deploy API Server / build-and-deploy (push) Successful in 39s
- Add ViolationRecord table (Prisma + migration) - CAPI POST /api/reports for user report submission - AAPI reports list + handle, violations list + penalty apply - Admin page: reports management + violation records tabs Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
90c27ee979
commit
a08fd4970a
@ -47,6 +47,7 @@ jobs:
|
||||
$MYSQL_CMD -e "DROP TABLE IF EXISTS ModelRoute;" 2>/dev/null || true
|
||||
$MYSQL_CMD -e "DROP TABLE IF EXISTS ProviderConfig;" 2>/dev/null || true
|
||||
$MYSQL_CMD -e "DROP TABLE IF EXISTS FallbackEvent;" 2>/dev/null || true
|
||||
$MYSQL_CMD -e "DROP TABLE IF EXISTS ViolationRecord;" 2>/dev/null || true
|
||||
$MYSQL_CMD -e "ALTER TABLE UploadedFile DROP COLUMN objectKey;" 2>/dev/null || true
|
||||
$MYSQL_CMD -e "ALTER TABLE UploadedFile DROP COLUMN bucket;" 2>/dev/null || true
|
||||
$MYSQL_CMD -e "DROP INDEX UploadedFile_objectKey_idx ON UploadedFile;" 2>/dev/null || true
|
||||
|
||||
@ -0,0 +1,16 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS `ViolationRecord` (
|
||||
`id` VARCHAR(191) NOT NULL,
|
||||
`userId` VARCHAR(191) NOT NULL,
|
||||
`contentType` VARCHAR(32) NOT NULL,
|
||||
`content` VARCHAR(1000) NOT NULL,
|
||||
`riskLevel` VARCHAR(16) NOT NULL,
|
||||
`penalty` VARCHAR(32) NOT NULL DEFAULT 'none',
|
||||
`appliedBy` VARCHAR(100) NULL,
|
||||
`appliedAt` DATETIME(3) NULL,
|
||||
`createdAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
|
||||
INDEX `ViolationRecord_userId_idx`(`userId`),
|
||||
INDEX `ViolationRecord_createdAt_idx`(`createdAt`),
|
||||
PRIMARY KEY (`id`)
|
||||
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
|
||||
@ -990,6 +990,21 @@ model ContentReport {
|
||||
@@index([createdAt])
|
||||
}
|
||||
|
||||
model ViolationRecord {
|
||||
id String @id @default(cuid())
|
||||
userId String
|
||||
contentType String @db.VarChar(32)
|
||||
content String @db.VarChar(1000)
|
||||
riskLevel String @db.VarChar(16)
|
||||
penalty String @default("none") @db.VarChar(32)
|
||||
appliedBy String? @db.VarChar(100)
|
||||
appliedAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
@@index([userId])
|
||||
@@index([createdAt])
|
||||
}
|
||||
|
||||
model ApiMetric {
|
||||
id String @id @default(cuid())
|
||||
path String @db.VarChar(255)
|
||||
|
||||
@ -1,20 +1,81 @@
|
||||
import { Controller, Get, Post, Delete, Body, Param, UseGuards } from '@nestjs/common';
|
||||
import { ApiTags, ApiBearerAuth } from '@nestjs/swagger';
|
||||
import { Controller, Get, Post, Delete, Body, Param, Query, UseGuards } from '@nestjs/common';
|
||||
import { ApiTags, ApiBearerAuth, ApiOperation } from '@nestjs/swagger';
|
||||
import { ContentSafetyService } from './content-safety.service';
|
||||
import { AdminAuthGuard } from '../../common/guards/admin-auth.guard';
|
||||
import { AdminRolesGuard } from '../../common/guards/admin-roles.guard';
|
||||
import { AdminRoles } from '../../common/decorators/admin-roles.decorator';
|
||||
import { Public } from '../../common/decorators/public.decorator';
|
||||
import type { AdminRole } from '../../common/types/admin-role.enum';
|
||||
|
||||
@ApiTags('content-safety')
|
||||
@Controller()
|
||||
export class ContentSafetyController {
|
||||
constructor(private readonly svc: ContentSafetyService) {}
|
||||
|
||||
// ═══ CAPI: User report submission ═══
|
||||
|
||||
@Public()
|
||||
@Post('reports')
|
||||
@ApiOperation({ summary: '用户提交举报' })
|
||||
async submitReport(@Body() d: { targetType: string; targetId: string; reason: string; reporterId?: string }) {
|
||||
const report = await this.svc.submitReport({
|
||||
reporterId: d.reporterId || 'anonymous',
|
||||
targetType: d.targetType,
|
||||
targetId: d.targetId,
|
||||
reason: d.reason,
|
||||
});
|
||||
return { success: true, data: report };
|
||||
}
|
||||
}
|
||||
|
||||
@ApiTags('admin-content-safety')
|
||||
@Controller('admin-api/content-safety')
|
||||
@UseGuards(AdminAuthGuard, AdminRolesGuard)
|
||||
@ApiBearerAuth()
|
||||
export class ContentSafetyController {
|
||||
export class AdminContentSafetyController {
|
||||
constructor(private readonly svc: ContentSafetyService) {}
|
||||
|
||||
@Get('words') @AdminRoles('SUPER_ADMIN' as AdminRole) async words() { return this.svc.getAllWords() }
|
||||
@Post('words') @AdminRoles('SUPER_ADMIN' as AdminRole) async addWord(@Body() d: { word: string; category: string; riskLevel: string }) { return this.svc.addWord(d.word, d.category, d.riskLevel) }
|
||||
@Delete('words/:id') @AdminRoles('SUPER_ADMIN' as AdminRole) async removeWord(@Param('id') id: string) { await this.svc.removeWord(id); return { success: true } }
|
||||
@Get('checks') @AdminRoles('SUPER_ADMIN' as AdminRole) async checks() { return this.svc.getChecks() }
|
||||
// ── Sensitive words ──
|
||||
|
||||
@Get('words') @AdminRoles('SUPER_ADMIN' as AdminRole)
|
||||
async words() { return this.svc.getAllWords() }
|
||||
|
||||
@Post('words') @AdminRoles('SUPER_ADMIN' as AdminRole)
|
||||
async addWord(@Body() d: { word: string; category: string; riskLevel: string }) { return this.svc.addWord(d.word, d.category, d.riskLevel) }
|
||||
|
||||
@Delete('words/:id') @AdminRoles('SUPER_ADMIN' as AdminRole)
|
||||
async removeWord(@Param('id') id: string) { await this.svc.removeWord(id); return { success: true } }
|
||||
|
||||
// ── AI safety checks log ──
|
||||
|
||||
@Get('checks') @AdminRoles('SUPER_ADMIN' as AdminRole)
|
||||
async checks() { return this.svc.getChecks() }
|
||||
|
||||
// ── Reports ──
|
||||
|
||||
@Get('reports') @AdminRoles('ADMIN' as AdminRole)
|
||||
@ApiOperation({ summary: '举报列表' })
|
||||
async reports(@Query('status') status?: string) {
|
||||
return this.svc.getReports(status);
|
||||
}
|
||||
|
||||
@Post('reports/:id/handle') @AdminRoles('ADMIN' as AdminRole)
|
||||
@ApiOperation({ summary: '处理举报' })
|
||||
async handleReport(@Param('id') id: string, @Body() d: { action: string; note?: string }) {
|
||||
return this.svc.handleReport(id, d.action, d.note);
|
||||
}
|
||||
|
||||
// ── Violations ──
|
||||
|
||||
@Get('violations') @AdminRoles('ADMIN' as AdminRole)
|
||||
@ApiOperation({ summary: '违规记录列表' })
|
||||
async violations(@Query('userId') userId?: string, @Query('limit') limit = '50') {
|
||||
return this.svc.getViolations(userId, parseInt(limit));
|
||||
}
|
||||
|
||||
@Post('violations/:id/penalty') @AdminRoles('SUPER_ADMIN' as AdminRole)
|
||||
@ApiOperation({ summary: '应用处罚' })
|
||||
async applyPenalty(@Param('id') id: string, @Body() d: { penalty: string }) {
|
||||
return this.svc.applyPenalty(id, d.penalty);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ContentSafetyController } from './content-safety.controller';
|
||||
import { ContentSafetyController, AdminContentSafetyController } from './content-safety.controller';
|
||||
import { ContentSafetyService } from './content-safety.service';
|
||||
import { PrismaService } from '../../infrastructure/database/prisma.service';
|
||||
import { RedisService } from '../../infrastructure/redis/redis.service';
|
||||
@ -8,7 +8,7 @@ import { AdminAuthGuard } from '../../common/guards/admin-auth.guard';
|
||||
import { AdminRolesGuard } from '../../common/guards/admin-roles.guard';
|
||||
|
||||
@Module({
|
||||
controllers: [ContentSafetyController],
|
||||
controllers: [ContentSafetyController, AdminContentSafetyController],
|
||||
providers: [ContentSafetyService, PrismaService, RedisService, QueueService, AdminAuthGuard, AdminRolesGuard],
|
||||
exports: [ContentSafetyService],
|
||||
})
|
||||
|
||||
@ -77,4 +77,56 @@ export class ContentSafetyService {
|
||||
|
||||
async getAllWords() { return this.prisma.sensitiveWord.findMany({ orderBy: { createdAt: 'desc' } }) }
|
||||
async getChecks(limit = 50) { return this.prisma.contentSafetyCheck.findMany({ orderBy: { createdAt: 'desc' }, take: limit }) }
|
||||
|
||||
// ── Reports ──
|
||||
|
||||
async submitReport(input: { reporterId: string; targetType: string; targetId: string; reason: string }) {
|
||||
return this.prisma.contentReport.create({ data: input });
|
||||
}
|
||||
|
||||
async getReports(status?: string) {
|
||||
return this.prisma.contentReport.findMany({
|
||||
where: status ? { status } : undefined,
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: 100,
|
||||
});
|
||||
}
|
||||
|
||||
async handleReport(id: string, action: string, note?: string) {
|
||||
const report = await this.prisma.contentReport.update({
|
||||
where: { id },
|
||||
data: { status: action, handleNote: note || null, handledAt: new Date() },
|
||||
});
|
||||
|
||||
// If confirmed violation, create violation record
|
||||
if (action === 'confirmed') {
|
||||
await this.prisma.violationRecord.create({
|
||||
data: {
|
||||
userId: report.reporterId,
|
||||
contentType: report.targetType,
|
||||
content: report.reason,
|
||||
riskLevel: 'medium',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
// ── Violations ──
|
||||
|
||||
async getViolations(userId?: string, limit = 50) {
|
||||
return this.prisma.violationRecord.findMany({
|
||||
where: userId ? { userId } : undefined,
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: limit,
|
||||
});
|
||||
}
|
||||
|
||||
async applyPenalty(id: string, penalty: string) {
|
||||
return this.prisma.violationRecord.update({
|
||||
where: { id },
|
||||
data: { penalty, appliedAt: new Date() },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -228,4 +228,64 @@ describe('M1 E2E Tests', () => {
|
||||
.expect([200, 201]);
|
||||
});
|
||||
});
|
||||
|
||||
// ══════════════════════════════════════════════
|
||||
// M1-04: Content Safety 深化
|
||||
// ══════════════════════════════════════════════
|
||||
describe('M1-04 Content Safety Deepening', () => {
|
||||
let token: string;
|
||||
beforeAll(async () => { token = await loginAdmin(); });
|
||||
|
||||
it('POST /api/reports → 201 submit user report', async () => {
|
||||
const res = await request(app.getHttpServer())
|
||||
.post('/api/reports')
|
||||
.send({ targetType: 'knowledge_item', targetId: 'test123', reason: '包含错误信息', reporterId: 'user1' })
|
||||
.expect([200, 201]);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
|
||||
it('GET /admin-api/content-safety/reports → 200 list reports', async () => {
|
||||
if (!token) return;
|
||||
const res = await request(app.getHttpServer())
|
||||
.get('/admin-api/content-safety/reports')
|
||||
.set('Authorization', `Bearer ${token}`)
|
||||
.expect(200);
|
||||
expect(Array.isArray(res.body.data)).toBe(true);
|
||||
});
|
||||
|
||||
it('GET /admin-api/content-safety/reports → 401 without token', async () => {
|
||||
await request(app.getHttpServer())
|
||||
.get('/admin-api/content-safety/reports')
|
||||
.expect(401);
|
||||
});
|
||||
|
||||
it('GET /admin-api/content-safety/violations → 200 list violations', async () => {
|
||||
if (!token) return;
|
||||
const res = await request(app.getHttpServer())
|
||||
.get('/admin-api/content-safety/violations')
|
||||
.set('Authorization', `Bearer ${token}`)
|
||||
.expect(200);
|
||||
expect(Array.isArray(res.body.data)).toBe(true);
|
||||
});
|
||||
|
||||
it('POST /admin-api/content-safety/violations/:id/penalty → apply penalty', async () => {
|
||||
if (!token) return;
|
||||
const res = await request(app.getHttpServer())
|
||||
.post('/admin-api/content-safety/violations/test-id/penalty')
|
||||
.set('Authorization', `Bearer ${token}`)
|
||||
.send({ penalty: 'warning' })
|
||||
.expect([200, 201]);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
|
||||
it('POST /admin-api/content-safety/reports/:id/handle → handle report', async () => {
|
||||
if (!token) return;
|
||||
const res = await request(app.getHttpServer())
|
||||
.post('/admin-api/content-safety/reports/test-id/handle')
|
||||
.set('Authorization', `Bearer ${token}`)
|
||||
.send({ action: 'dismissed', note: '已处理' })
|
||||
.expect([200, 201]);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -90,6 +90,7 @@ const modelNames = [
|
||||
'contentSafetyCheck', 'contentReport', 'apiMetric', 'taskLog',
|
||||
'userMembership', 'quotaUsage', 'costDailySummary', 'secretRecord',
|
||||
'secretAccessLog', 'modelRoute', 'providerConfig', 'fallbackEvent',
|
||||
'violationRecord', 'contentReport',
|
||||
]
|
||||
|
||||
for (const name of modelNames) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user