Change crypto::aes_t to variable size and cleanup some crypto code

src/crypto.cpp

@@ -152,10 +152,11 @@ namespace crypto {
       auto cipher = tagged_cipher.substr(tag_size);
       auto tag = tagged_cipher.substr(0, tag_size);
 
-      plaintext.resize((cipher.size() + 15) / 16 * 16);
+      plaintext.resize(round_to_pkcs7_padded(cipher.size()));
 
-      int size;
+      int update_outlen, final_outlen;
-      if (EVP_DecryptUpdate(decrypt_ctx.get(), plaintext.data(), &size, (const std::uint8_t *) cipher.data(), cipher.size()) != 1) {
+
+      if (EVP_DecryptUpdate(decrypt_ctx.get(), plaintext.data(), &update_outlen, (const std::uint8_t *) cipher.data(), cipher.size()) != 1) {
         return -1;
       }
 
@@ -163,12 +164,11 @@ namespace crypto {
         return -1;
       }
 
-      int len = size;
+      if (EVP_DecryptFinal_ex(decrypt_ctx.get(), plaintext.data() + update_outlen, &final_outlen) != 1) {
-      if (EVP_DecryptFinal_ex(decrypt_ctx.get(), plaintext.data() + size, &len) != 1) {
         return -1;
       }
 
-      plaintext.resize(size + len);
+      plaintext.resize(update_outlen + final_outlen);
       return 0;
     }
 
@@ -187,16 +187,15 @@ namespace crypto {
       auto tag = tagged_cipher;
       auto cipher = tag + tag_size;
 
-      int len;
+      int update_outlen, final_outlen;
-      int size = round_to_pkcs7_padded(plaintext.size());
 
       // Encrypt into the caller's buffer
-      if (EVP_EncryptUpdate(encrypt_ctx.get(), cipher, &size, (const std::uint8_t *) plaintext.data(), plaintext.size()) != 1) {
+      if (EVP_EncryptUpdate(encrypt_ctx.get(), cipher, &update_outlen, (const std::uint8_t *) plaintext.data(), plaintext.size()) != 1) {
         return -1;
       }
 
       // GCM encryption won't ever fill ciphertext here but we have to call it anyway
-      if (EVP_EncryptFinal_ex(encrypt_ctx.get(), cipher + size, &len) != 1) {
+      if (EVP_EncryptFinal_ex(encrypt_ctx.get(), cipher + update_outlen, &final_outlen) != 1) {
         return -1;
       }
 
@@ -204,13 +203,11 @@ namespace crypto {
         return -1;
       }
 
-      return len + size;
+      return update_outlen + final_outlen;
     }
 
     int
     ecb_t::decrypt(const std::string_view &cipher, std::vector<std::uint8_t> &plaintext) {
-      int len;
-
       auto fg = util::fail_guard([this]() {
         EVP_CIPHER_CTX_reset(decrypt_ctx.get());
       });
@@ -221,19 +218,19 @@ namespace crypto {
       }
 
       EVP_CIPHER_CTX_set_padding(decrypt_ctx.get(), padding);
+      plaintext.resize(round_to_pkcs7_padded(cipher.size()));
 
-      plaintext.resize((cipher.size() + 15) / 16 * 16);
+      int update_outlen, final_outlen;
-      auto size = (int) plaintext.size();
+
-      // Decrypt into the caller's buffer, leaving room for the auth tag to be prepended
+      if (EVP_DecryptUpdate(decrypt_ctx.get(), plaintext.data(), &update_outlen, (const std::uint8_t *) cipher.data(), cipher.size()) != 1) {
-      if (EVP_DecryptUpdate(decrypt_ctx.get(), plaintext.data(), &size, (const std::uint8_t *) cipher.data(), cipher.size()) != 1) {
         return -1;
       }
 
-      if (EVP_DecryptFinal_ex(decrypt_ctx.get(), plaintext.data(), &len) != 1) {
+      if (EVP_DecryptFinal_ex(decrypt_ctx.get(), plaintext.data() + update_outlen, &final_outlen) != 1) {
         return -1;
       }
 
-      plaintext.resize(len + size);
+      plaintext.resize(update_outlen + final_outlen);
       return 0;
     }
 
@@ -249,22 +246,20 @@ namespace crypto {
       }
 
       EVP_CIPHER_CTX_set_padding(encrypt_ctx.get(), padding);
+      cipher.resize(round_to_pkcs7_padded(plaintext.size()));
 
-      int len;
+      int update_outlen, final_outlen;
-
-      cipher.resize((plaintext.size() + 15) / 16 * 16);
-      auto size = (int) cipher.size();
 
       // Encrypt into the caller's buffer
-      if (EVP_EncryptUpdate(encrypt_ctx.get(), cipher.data(), &size, (const std::uint8_t *) plaintext.data(), plaintext.size()) != 1) {
+      if (EVP_EncryptUpdate(encrypt_ctx.get(), cipher.data(), &update_outlen, (const std::uint8_t *) plaintext.data(), plaintext.size()) != 1) {
         return -1;
       }
 
-      if (EVP_EncryptFinal_ex(encrypt_ctx.get(), cipher.data() + size, &len) != 1) {
+      if (EVP_EncryptFinal_ex(encrypt_ctx.get(), cipher.data() + update_outlen, &final_outlen) != 1) {
         return -1;
       }
 
-      cipher.resize(len + size);
+      cipher.resize(update_outlen + final_outlen);
       return 0;
     }
 
@@ -280,20 +275,18 @@ namespace crypto {
         return false;
       }
 
-      int len;
+      int update_outlen, final_outlen;
-
-      int size = plaintext.size();  // round_to_pkcs7_padded(plaintext.size());
 
       // Encrypt into the caller's buffer
-      if (EVP_EncryptUpdate(encrypt_ctx.get(), cipher, &size, (const std::uint8_t *) plaintext.data(), plaintext.size()) != 1) {
+      if (EVP_EncryptUpdate(encrypt_ctx.get(), cipher, &update_outlen, (const std::uint8_t *) plaintext.data(), plaintext.size()) != 1) {
         return -1;
       }
 
-      if (EVP_EncryptFinal_ex(encrypt_ctx.get(), cipher + size, &len) != 1) {
+      if (EVP_EncryptFinal_ex(encrypt_ctx.get(), cipher + update_outlen, &final_outlen) != 1) {
         return -1;
       }
 
-      return size + len;
+      return update_outlen + final_outlen;
     }
 
     ecb_t::ecb_t(const aes_t &key, bool padding):
@@ -309,7 +302,7 @@ namespace crypto {
 
   aes_t
   gen_aes_key(const std::array<uint8_t, 16> &salt, const std::string_view &pin) {
-    aes_t key;
+    aes_t key(16);
 
     std::string salt_pin;
     salt_pin.reserve(salt.size() + pin.size());

src/crypto.h

@@ -23,7 +23,7 @@ namespace crypto {
 
   using sha256_t = std::array<std::uint8_t, SHA256_DIGEST_LENGTH>;
 
-  using aes_t = std::array<std::uint8_t, 16>;
+  using aes_t = std::vector<std::uint8_t>;
   using x509_t = util::safe_ptr<X509, X509_free>;
   using x509_store_t = util::safe_ptr<X509_STORE, X509_STORE_free>;
   using x509_store_ctx_t = util::safe_ptr<X509_STORE_CTX, X509_STORE_CTX_free>;

src/nvhttp.cpp

@@ -271,8 +271,10 @@ namespace nvhttp {
   make_launch_session(bool host_audio, const args_t &args) {
     rtsp_stream::launch_session_t launch_session;
 
+    auto rikey = util::from_hex_vec(get_arg(args, "rikey"), true);
+    std::copy(rikey.cbegin(), rikey.cend(), std::back_inserter(launch_session.gcm_key));
+
     launch_session.host_audio = host_audio;
-    launch_session.gcm_key = util::from_hex<crypto::aes_t>(get_arg(args, "rikey"), true);
     std::stringstream mode = std::stringstream(get_arg(args, "mode", "0x0x0"));
     // Split mode by the char "x", to populate width/height/fps
     int x = 0;
@@ -296,11 +298,10 @@ namespace nvhttp {
     launch_session.av_ping_payload = util::hex_vec(raw_payload);
     RAND_bytes((unsigned char *) &launch_session.control_connect_data, sizeof(launch_session.control_connect_data));
 
+    launch_session.iv.resize(16);
     uint32_t prepend_iv = util::endian::big<uint32_t>(util::from_view(get_arg(args, "rikeyid")));
     auto prepend_iv_p = (uint8_t *) &prepend_iv;
-
+    std::copy(prepend_iv_p, prepend_iv_p + sizeof(prepend_iv), std::begin(launch_session.iv));
-    auto next = std::copy(prepend_iv_p, prepend_iv_p + sizeof(prepend_iv), std::begin(launch_session.iv));
-    std::fill(next, std::end(launch_session.iv), 0);
     return launch_session;
   }
 

src/stream.cpp

@@ -242,7 +242,7 @@ namespace stream {
       return plaintext.size();
     }
 
-    crypto::aes_t iv {};
+    crypto::aes_t iv(16);
     *(std::uint32_t *) iv.data() = util::endian::big<std::uint32_t>(avRiKeyIv + destination->rtp.sequenceNumber);
 
     return cbc.encrypt(std::string_view { (char *) std::begin(plaintext), plaintext.size() }, destination->payload(), &iv);
@@ -379,7 +379,7 @@ namespace stream {
 
     struct {
       crypto::cipher::gcm_t cipher;
-      crypto::aes_t iv;
+      crypto::aes_t legacy_input_enc_iv;  // Only used when the client doesn't support full control stream encryption
 
       uint32_t connect_data;  // Used for new clients with ML_FF_SESSION_ID_V1
       std::string expected_peer_address;  // Only used for legacy clients without ML_FF_SESSION_ID_V1
@@ -414,7 +414,7 @@ namespace stream {
       return plaintext;
     }
 
-    crypto::aes_t iv {};
+    crypto::aes_t iv(16);
     auto seq = session->control.seq++;
     iv[0] = seq;
 
@@ -881,7 +881,7 @@ namespace stream {
       std::vector<uint8_t> plaintext;
 
       auto &cipher = session->control.cipher;
-      auto &iv = session->control.iv;
+      auto &iv = session->control.legacy_input_enc_iv;
       if (cipher.decrypt(tagged_cipher, plaintext, &iv)) {
         // something went wrong :(
 
@@ -891,7 +891,7 @@ namespace stream {
         return;
       }
 
-      if (tagged_cipher_length >= 16 + sizeof(crypto::aes_t)) {
+      if (tagged_cipher_length >= 16 + iv.size()) {
         std::copy(payload.end() - 16, payload.end(), std::begin(iv));
       }
 
@@ -915,7 +915,7 @@ namespace stream {
       std::string_view tagged_cipher { (char *) header->payload(), (size_t) tagged_cipher_length };
 
       auto &cipher = session->control.cipher;
-      crypto::aes_t iv {};
+      crypto::aes_t iv(16);
       iv[0] = (std::uint8_t) seq;
 
       // update control sequence
@@ -1791,7 +1791,7 @@ namespace stream {
       session->control.connect_data = launch_session.control_connect_data;
       session->control.feedback_queue = mail->queue<platf::gamepad_feedback_msg_t>(mail::gamepad_feedback);
       session->control.hdr_queue = mail->event<video::hdr_info_t>(mail::hdr);
-      session->control.iv = launch_session.iv;
+      session->control.legacy_input_enc_iv = launch_session.iv;
       session->control.cipher = crypto::cipher::gcm_t {
         launch_session.gcm_key, false
       };