[video_core] fix std::bitset<> dirty tracker OOB, fix slightly wrong index format estimate (#4006)

u8 may have been 0xff, (aka. 255), but bitset was only 255 elements, so doing bitset[255] is technically OOB

additionally the max size estimate for index formats was not correct, there can be up to 256 elements with a u8 format index, not just 255

Signed-off-by: lizzie <lizzie@eden-emu.dev>

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4006
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

src/video_core/engines/maxwell_3d.h

@@ -2215,7 +2215,7 @@ public:
             u32 first;
             u32 count;
 
-            unsigned FormatSizeInBytes() const {
+            size_t FormatSizeInBytes() const {
                 switch (format) {
                 case IndexFormat::UnsignedByte:
                     return 1;
@@ -2224,7 +2224,7 @@ public:
                 case IndexFormat::UnsignedInt:
                     return 4;
                 }
-                ASSERT(false);
+                UNREACHABLE();
                 return 1;
             }
 
@@ -3148,9 +3148,9 @@ public:
     }
 
     struct DirtyState {
-        using Flags = std::bitset<(std::numeric_limits<u8>::max)()>;
+        using Flags = std::bitset<(std::numeric_limits<u8>::max)() + 1>;
         using Table = std::array<u8, Regs::NUM_REGS>;
-        using Tables = std::array<Table, 2>;
+        using Tables = std::array<std::array<u8, Regs::NUM_REGS>, 2>;
 
         Flags flags;
         Tables tables{};