simple encryptor

• platform: hackthebox
• category: reverse engineering
• difficulty: easy

scenario

on our regular checkups of our secret flag storage server we found out that we were hit by ransomware!
the original flag data is nowhere to be found, but luckily we not only have the encrypted file but also the encryption program itself.

challenge

decompile the executable with hex-rays and go to the main function:

int __fastcall main(int argc, const char **argv, const char **envp)
{
  char v3; // al
  char v5; // [rsp+7h] [rbp-39h]
  unsigned int seed[2]; // [rsp+8h] [rbp-38h] BYREF
  signed __int64 i; // [rsp+10h] [rbp-30h]
  FILE *stream; // [rsp+18h] [rbp-28h]
  size_t size; // [rsp+20h] [rbp-20h]
  void *ptr; // [rsp+28h] [rbp-18h]
  FILE *s; // [rsp+30h] [rbp-10h]
  unsigned __int64 v12; // [rsp+38h] [rbp-8h]

  v12 = __readfsqword(0x28u);
  stream = fopen("flag", "rb");
  fseek(stream, 0, 2);
  size = ftell(stream);
  fseek(stream, 0, 0);
  ptr = malloc(size);
  fread(ptr, size, 1u, stream);
  fclose(stream);
  seed[0] = time(nullptr);
  srand(seed[0]);
  for ( i = 0; i < (__int64)size; ++i )
  {
    *((_BYTE *)ptr + i) ^= rand();
    v3 = rand();
    v5 = *((_BYTE *)ptr + i);
    seed[1] = v3 & 7;
    *((_BYTE *)ptr + i) = __ROL1__(v5, v3 & 7);
  }
  s = fopen("flag.enc", "wb");
  fwrite(seed, 1u, 4u, s);
  fwrite(ptr, 1u, size, s);
  fclose(s);
  return 0;
}

clean code

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <stdint.h>

static inline uint8_t rol8(uint8_t val, int shift) {
  shift &= 7;
  if (shift == 0) return val;
  return (uint8_t) ((val << shift) | (val >> (8 - shift)));
}

int main(void) {

  FILE *stream = fopen("flag", "rb");
  fseek(stream, 0, SEEK_END);
  long size = ftell(stream);
  fseek(stream, 0, SEEK_SET);
  uint8_t *data = malloc(size);
  fread(data, size, 1, stream);
  fclose(stream);

  uint32_t seed = (uint32_t) time(NULL);
  srand(seed);

  for (long i = 0; i < size; i++) {
    data[i] ^= (uint8_t) rand();
    int shift = rand() & 7;
    data[i] = rol8(data[i], shift);
  }

  FILE *out = fopen("flag.enc", "wb");
  fwrite(&seed, 1, 4, out);
  fwrite(data, 1, size, out);
  fclose(out);

  free(data);

  return 0;
}

exploit

reversing the logic

• take the 4 first bytes from flag.enc and srand() it
• generate xor key with rand()
• generate the shift value
• rotate 8 bytes right from shift
• xor with key
  

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <stdint.h>

static inline uint8_t ror8(uint8_t val, int shift) {
  shift &= 7;
  if (shift == 0) return val;
  return (uint8_t) ((val >> shift) | (val << (8 - shift)));
}

int main(void) {

  FILE *stream = fopen("flag.enc", "rb");
  uint32_t seed;
  fread(&seed, 1, 4, stream);

  long pos = ftell(stream);
  fseek(stream, 0, SEEK_END);
  long size = ftell(stream) - pos;
  fseek(stream, pos, SEEK_SET);

  uint8_t *data = malloc(size);
  fread(data, size, 1, stream);
  fclose(stream);

  srand(seed);

  for (long i = 0; i < size; i++) {

    uint8_t key = rand();
    uint8_t shift = rand() & 7;
    data[i] = ror8(data[i], shift);
    data[i] ^= key;
  }

  FILE *out = fopen("flag.dec", "wb");
  fwrite(data, 1, size, out);
  fclose(out);

  free(data);

  return 0;
}

whoami – uses – writeups – music – reading