diff --git a/sponsorblock_lm.lua b/sponsorblock_lm.lua
index 238d6c9..cab6a6a 100644
--- a/sponsorblock_lm.lua
+++ b/sponsorblock_lm.lua
@@ -84,14 +84,26 @@ function observePlaybackTime(_, current_pos)
                 mp.set_property_number("playback-time", segment.segment[2])
                 current_pos = segment.segment[2]
             elseif segment.actionType == "mute" then
-                local muted = mp.get_property_bool("ao-mute") or mp.get_property_bool("mute")
-                if muted ~= true and (video_data.muted_due_to_segment == nil
-                        or segment.segment[1] ~= video_data.muted_due_to_segment[1]
-                        or segment.segment[2] ~= video_data.muted_due_to_segment[2]) then
+                -- if ao-mute or mute is false, then it's not muted, which causes it to be muted
+                local caused_mute = (mp.get_property_bool("ao-mute") or mp.get_property_bool("mute")) == false
+                -- if subs are visible, they'll be hidden
+                local caused_hidden_subs = mp.get_property_bool("sub-visibility") == true
+                if (caused_mute or caused_hidden_subs) and (video_data.current_mute_segment == nil
+                        or segment.segment[1] ~= video_data.current_mute_segment.segment[1]
+                        or segment.segment[2] ~= video_data.current_mute_segment.segment[2]) then
                     mp.osd_message("[sponsorblock] Muting " .. segment.segment[2] - current_pos
                             .. "s " .. category_name)
-                    video_data.muted_due_to_segment = segment.segment
-                    mp.set_property_bool("mute", true)
+                    video_data.current_mute_segment = {
+                        segment = segment.segment,
+                        caused_mute = caused_mute,
+                        caused_hidden_subs = caused_hidden_subs
+                    }
+                    if caused_mute then
+                        mp.set_property_bool("mute", true)
+                    end
+                    if caused_hidden_subs then
+                        mp.set_property_bool("sub-visibility", false)
+                    end
                 end
             else
                 mp.msg.warn("Unknown action type:", segment.actionType)
@@ -99,17 +111,27 @@ function observePlaybackTime(_, current_pos)
         end
     end
 
-    if video_data.muted_due_to_segment ~= nil and
-            (current_pos < video_data.muted_due_to_segment[1] or
-            video_data.muted_due_to_segment[2] <= current_pos) then
-        mp.set_property_bool("mute", false)
-        video_data.muted_due_to_segment = nil
+    if video_data.current_mute_segment ~= nil and
+            (current_pos < video_data.current_mute_segment.segment[1] or
+            video_data.current_mute_segment.segment[2] <= current_pos) then
+        if video_data.current_mute_segment.caused_mute then
+            mp.set_property_bool("mute", false)
+        end
+        if video_data.current_mute_segment.caused_hidden_subs then
+            mp.set_property_bool("sub-visibility", true)
+        end
+        video_data.current_mute_segment = nil
     end
 end
 
 function endFile()
-    if video_data ~= nil and video_data.muted_due_to_segment ~= nil then
-        mp.set_property_bool("mute", false)
+    if video_data ~= nil and video_data.current_mute_segment ~= nil then
+        if video_data.current_mute_segment.caused_mute then
+            mp.set_property_bool("mute", false)
+        end
+        if video_data.current_mute_segment.caused_hidden_subs then
+            mp.set_property_bool("sub-visibility", true)
+        end
     end
     video_data = nil
 end
@@ -121,9 +143,14 @@ function toggleSponsorblock()
         message = message .. "Enabled"
     else
         disabled = true
-        if video_data ~= nil and video_data.muted_due_to_segment ~= nil then
-            mp.set_property_bool("mute", false)
-            video_data.muted_due_to_segment = nil
+        if video_data ~= nil and video_data.current_mute_segment ~= nil then
+            if video_data.current_mute_segment.caused_mute then
+                mp.set_property_bool("mute", false)
+            end
+            if video_data.current_mute_segment.caused_hidden_subs then
+                mp.set_property_bool("sub-visibility", true)
+            end
+            video_data.current_mute_segment = nil
         end
         message = message .. "Disabled"
     end
@@ -193,7 +220,6 @@ function getSegments(video_id)
 
     for _, i in pairs(segments) do
         if i.videoID == video_id then
-            i.muted_due_to_segment = nil
             return i
         end
     end