Data Processing & Validation

// Example: Data Processing Pipeline
class DataProcessor {
  constructor(config) {
    this.validators = config.validators || [];
    this.transformers = config.transformers || [];
    this.storage = config.storage;
    this.errorHandler = config.errorHandler;
  }

  async processData(rawData, options = {}) {
    const processingId = this.generateProcessingId();
    const startTime = Date.now();

    try {
      // Step 1: Initial validation
      const validationResult = await this.validateData(rawData);
      if (!validationResult.isValid) {
        throw new Error(`Validation failed: ${validationResult.errors.join(', ')}`);
      }

      // Step 2: Data transformation
      let processedData = rawData;
      for (const transformer of this.transformers) {
        processedData = await transformer.transform(processedData);
      }

      // Step 3: Final validation
      const finalValidation = await this.validateProcessedData(processedData);
      if (!finalValidation.isValid) {
        throw new Error(`Final validation failed: ${finalValidation.errors.join(', ')}`);
      }

      // Step 4: Storage
      const storageResult = await this.storage.save(processedData, {
        processingId,
        timestamp: new Date(),
        metadata: options.metadata
      });

      // Step 5: Audit logging
      await this.logProcessingResult({
        processingId,
        status: 'success',
        duration: Date.now() - startTime,
        recordCount: Array.isArray(processedData) ? processedData.length : 1,
        storageId: storageResult.id
      });

      return {
        success: true,
        processingId,
        data: processedData,
        metadata: {
          duration: Date.now() - startTime,
          recordCount: Array.isArray(processedData) ? processedData.length : 1
        }
      };
    } catch (error) {
      await this.handleProcessingError(error, processingId, rawData);
      throw error;
    }
  }

  async validateData(data) {
    const errors = [];

    for (const validator of this.validators) {
      try {
        const result = await validator.validate(data);
        if (!result.isValid) {
          errors.push(...result.errors);
        }
      } catch (error) {
        errors.push(`Validator error: ${error.message}`);
      }
    }

    return { isValid: errors.length === 0, errors };
  }

  generateProcessingId() {
    return `proc_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
  }
}

// Usage Example
const processor = new DataProcessor({
  validators: [
    new SchemaValidator(dataSchema),
    new BusinessRuleValidator(businessRules)
  ],
  transformers: [
    new DataCleaner(),
    new DataEnricher(),
    new DataNormalizer()
  ],
  storage: new DatabaseStorage(),
  errorHandler: new ProcessingErrorHandler()
});

async function processIncomingData(rawData) {
  try {
    const result = await processor.processData(rawData, {
      metadata: { source: 'api', version: '1.0' }
    });
    console.log('Processing completed:', result.processingId);
    return result;
  } catch (error) {
    console.error('Processing failed:', error.message);
    throw error;
  }
}